如何写一个图片去重脚本

wen 实用脚本 2026-08-02 240

本文目录导读：

如何写一个图片去重脚本

方案一：基于文件哈希的简单去重（最快）
方案二：基于图像特征的感知哈希（能检测相似图片）
方案三：完整的去重工具（带GUI界面）
使用说明
安装依赖
注意事项

我来帮你写一个图片去重脚本，这里提供几种不同的方案,从简单到复杂：

基于文件哈希的简单去重（最快）

import os
import hashlib
from collections import defaultdict
def get_file_hash(filepath, chunk_size=8192):
    """计算文件哈希值"""
    hash_md5 = hashlib.md5()
    with open(filepath, 'rb') as f:
        for chunk in iter(lambda: f.read(chunk_size), b''):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()
def find_duplicate_files(folder_path):
    """查找重复文件"""
    hash_dict = defaultdict(list)
    # 遍历文件夹
    for root, dirs, files in os.walk(folder_path):
        for filename in files:
            filepath = os.path.join(root, filename)
            if filename.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
                try:
                    file_hash = get_file_hash(filepath)
                    hash_dict[file_hash].append(filepath)
                except Exception as e:
                    print(f"Error processing {filepath}: {e}")
    # 找出重复文件
    duplicates = {hash_val: paths for hash_val, paths in hash_dict.items() if len(paths) > 1}
    return duplicates
def delete_duplicates(duplicates, keep_first=True):
    """删除重复文件"""
    deleted_count = 0
    for hash_val, paths in duplicates.items():
        if keep_first:
            # 保留第一个，删除其余
            for filepath in paths[1:]:
                try:
                    os.remove(filepath)
                    print(f"Deleted: {filepath}")
                    deleted_count += 1
                except Exception as e:
                    print(f"Error deleting {filepath}: {e}")
        else:
            # 全部删除
            for filepath in paths:
                try:
                    os.remove(filepath)
                    print(f"Deleted: {filepath}")
                    deleted_count += 1
                except Exception as e:
                    print(f"Error deleting {filepath}: {e}")
    print(f"Total deleted: {deleted_count}")
# 使用示例
if __name__ == "__main__":
    target_folder = "path/to/your/images"
    duplicates = find_duplicate_files(target_folder)
    if duplicates:
        print("Found duplicates:")
        for hash_val, paths in duplicates.items():
            print(f"Hash: {hash_val}")
            for path in paths:
                print(f"  - {path}")
        # 询问是否删除
        choice = input("Delete duplicates? (y/n): ").lower()
        if choice == 'y':
            delete_duplicates(duplicates)
    else:
        print("No duplicates found!")

基于图像特征的感知哈希（能检测相似图片）

import os
import hashlib
from PIL import Image
import numpy as np
from collections import defaultdict
def average_hash(image_path, hash_size=16):
    """计算图片的平均哈希值"""
    try:
        img = Image.open(image_path)
        img = img.resize((hash_size, hash_size), Image.LANCZOS)
        img = img.convert('L')  # 转为灰度
        pixels = np.array(img)
        avg = pixels.mean()
        # 生成哈希
        diff = pixels > avg
        hash_str = ''.join(str(int(x)) for x in diff.flatten())
        return hash_str
    except Exception as e:
        print(f"Error processing {image_path}: {e}")
        return None
def find_similar_images(folder_path, threshold=20):
    """查找相似图片"""
    image_hashes = []
    for root, dirs, files in os.walk(folder_path):
        for filename in files:
            if filename.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
                filepath = os.path.join(root, filename)
                hash_str = average_hash(filepath)
                if hash_str:
                    image_hashes.append((filepath, hash_str))
    # 找出相似的图片
    duplicates = []
    processed = set()
    for i, (filepath1, hash1) in enumerate(image_hashes):
        if i in processed:
            continue
        similar_group = [filepath1]
        for j, (filepath2, hash2) in enumerate(image_hashes):
            if i != j and j not in processed:
                # 计算汉明距离
                hamming_dist = sum(c1 != c2 for c1, c2 in zip(hash1, hash2))
                if hamming_dist < threshold:
                    similar_group.append(filepath2)
                    processed.add(j)
        if len(similar_group) > 1:
            duplicates.append(similar_group)
    return duplicates
# 使用示例
if __name__ == "__main__":
    target_folder = "path/to/your/images"
    print("Finding similar images...")
    similar_groups = find_similar_images(target_folder)
    if similar_groups:
        print("Found similar image groups:")
        for group in similar_groups:
            print("Group:")
            for path in group:
                print(f"  - {path}")
        choice = input("Remove similar images (keep first of each group)? (y/n): ").lower()
        if choice == 'y':
            for group in similar_groups:
                for path in group[1:]:
                    try:
                        os.remove(path)
                        print(f"Deleted: {path}")
                    except Exception as e:
                        print(f"Error deleting {path}: {e}")
    else:
        print("No similar images found!")

完整的去重工具（带GUI界面）

import os
import hashlib
import tkinter as tk
from tkinter import filedialog, messagebox, ttk
from PIL import Image, ImageTk
import threading
class ImageDedupTool:
    def __init__(self, root):
        self.root = root
        self.root.title("图片去重工具")
        self.root.geometry("800x600")
        self.folder_path = None
        self.duplicates = {}
        # 创建界面元素
        self.setup_ui()
    def setup_ui(self):
        # 顶部按钮框架
        top_frame = tk.Frame(self.root)
        top_frame.pack(pady=10)
        tk.Button(top_frame, text="选择文件夹", command=self.select_folder).pack(side=tk.LEFT, padx=5)
        tk.Button(top_frame, text="查找重复", command=self.find_duplicates).pack(side=tk.LEFT, padx=5)
        tk.Button(top_frame, text="删除重复", command=self.delete_duplicates).pack(side=tk.LEFT, padx=5)
        # 进度条
        self.progress = ttk.Progressbar(self.root, length=500, mode='indeterminate')
        self.progress.pack(pady=10)
        # 结果列表
        self.tree = ttk.Treeview(self.root, columns=('type', 'path'), show="tree")
        self.tree.pack(fill=tk.BOTH, expand=True, padx=10, pady=10)
        # 状态栏
        self.status_label = tk.Label(self.root, text="Ready")
        self.status_label.pack(side=tk.BOTTOM, pady=5)
    def select_folder(self):
        self.folder_path = filedialog.askdirectory()
        if self.folder_path:
            self.status_label.config(text=f"Selected folder: {self.folder_path}")
    def find_duplicates(self):
        if not self.folder_path:
            messagebox.showerror("Error", "请先选择文件夹！")
            return
        self.progress.start()
        threading.Thread(target=self.find_duplicates_worker).start()
    def find_duplicates_worker(self):
        hash_dict = {}
        for root, dirs, files in os.walk(self.folder_path):
            for filename in files:
                if filename.lower().endswith(('.jpg', '.jpeg', '.png', '.gif', '.bmp')):
                    filepath = os.path.join(root, filename)
                    try:
                        file_hash = self.calculate_md5(filepath)
                        if file_hash in hash_dict:
                            hash_dict[file_hash].append(filepath)
                        else:
                            hash_dict[file_hash] = [filepath]
                    except Exception as e:
                        self.root.after(0, self.update_status, f"Error: {filepath}")
        self.duplicates = {h: paths for h, paths in hash_dict.items() if len(paths) > 1}
        self.root.after(0, self.display_results)
        self.root.after(0, self.progress.stop)
    def calculate_md5(self, filepath):
        hash_md5 = hashlib.md5()
        with open(filepath, 'rb') as f:
            for chunk in iter(lambda: f.read(8192), b''):
                hash_md5.update(chunk)
        return hash_md5.hexdigest()
    def display_results(self):
        # 清空列表
        for item in self.tree.get_children():
            self.tree.delete(item)
        if not self.duplicates:
            messagebox.showinfo("Info", "没有找到重复文件！")
            return
        for hash_val, paths in self.duplicates.items():
            parent = self.tree.insert('', 'end', text=f"重复组 (Hash: {hash_val[:8]}...)", open=True)
            for path in paths:
                self.tree.insert(parent, 'end', text=path)
        self.update_status(f"找到 {len(self.duplicates)} 组重复文件")
    def delete_duplicates(self):
        if not self.duplicates:
            messagebox.showinfo("Info", "没有找到重复文件！")
            return
        deleted_count = 0
        for hash_val, paths in self.duplicates.items():
            # 保留每组中的第一个，删除其余
            for path in paths[1:]:
                try:
                    os.remove(path)
                    deleted_count += 1
                except Exception as e:
                    self.update_status(f"删除失败: {path} - {e}")
        self.update_status(f"Successfully deleted {deleted_count} files")
        messagebox.showinfo("Info", f"成功删除 {deleted_count} 个重复文件")
        self.find_duplicates()  # 刷新显示
    def update_status(self, msg):
        self.status_label.config(text=msg)
# 运行应用
if __name__ == "__main__":
    root = tk.Tk()
    app = ImageDedupTool(root)
    root.mainloop()

使用说明

方案一是最简单的基于MD5的完全重复检测
方案二使用感知哈希，可以检测经过轻微修改（如旋转、缩放）的相似图片
方案三提供GUI界面，更加友好

安装依赖

# 基础版本
pip install os-sys
# 如果需要感知哈希或GUI功能
pip install pillow numpy tkinter

注意事项

使用前建议先备份重要图片
删除操作不可逆，请谨慎操作
可以根据需要调整哈希阈值和文件类型

选择适合你需求的方案,如果有特殊要求可以进一步调整代码！

本文地址： https://www.bmjup.com/post/36521.html