本文目录导读：
使用Python + 图像哈希（简单有效）
import os
import shutil
from PIL import Image
import imagehash
from collections import defaultdict
def find_similar_images(folder_path, hash_size=8, threshold=5):
"""
使用感知哈希查找相似图片
Args:
folder_path: 图片文件夹路径
hash_size: 哈希大小（默认8，越小越宽松）
threshold: 汉明距离阈值（越小越相似）
"""
similar_groups = defaultdict(list)
processed = set()
# 获取所有图片文件
image_extensions = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.webp'}
images = [f for f in os.listdir(folder_path)
if os.path.splitext(f)[1].lower() in image_extensions]
# 计算每张图片的哈希值
for i, img_file in enumerate(images):
if img_file in processed:
continue
img_path = os.path.join(folder_path, img_file)
try:
with Image.open(img_path) as img:
# 使用感知哈希
phash = imagehash.phash(img, hash_size=hash_size)
similar_groups[phash].append(img_file)
processed.add(img_file)
# 与其他图片比较
for other_file in images[i+1:]:
if other_file in processed:
continue
other_path = os.path.join(folder_path, other_file)
try:
with Image.open(other_path) as other_img:
other_hash = imagehash.phash(other_img, hash_size=hash_size)
if phash - other_hash <= threshold:
similar_groups[phash].append(other_file)
processed.add(other_file)
except Exception as e:
print(f"处理 {other_file} 时出错: {e}")
except Exception as e:
print(f"处理 {img_file} 时出错: {e}")
# 返回相似图片组（过滤掉单独出现的）
return {k: v for k, v in similar_groups.items() if len(v) > 1}
# 使用示例
folder = "your_image_folder"
similar_images = find_similar_images(folder, hash_size=8, threshold=5)
for hash_value, images in similar_images.items():
print(f"相似组 ({len(images)}张):")
for img in images:
print(f" - {img}")
print("---")
使用OpenCV进行特征匹配（更精确）
import cv2
import os
import numpy as np
from sklearn.cluster import DBSCAN
class ImageSimilarityFinder:
def __init__(self):
# 使用ORB特征检测器
self.orb = cv2.ORB_create(nfeatures=500)
self.bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
def extract_features(self, image_path):
"""提取图片特征"""
img = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)
if img is None:
return None
# 调整大小以提高性能
img = cv2.resize(img, (300, 300))
keypoints, descriptors = self.orb.detectAndCompute(img, None)
return descriptors
def calculate_similarity(self, desc1, desc2):
"""计算两张图片的相似度"""
if desc1 is None or desc2 is None:
return 0
matches = self.bf.match(desc1, desc2)
# 根据匹配数量计算相似度
if len(matches) > 0:
similarity = len(matches) / max(len(desc1), len(desc2))
return similarity
return 0
def find_similar_groups(self, folder_path, similarity_threshold=0.3):
"""查找相似图片组"""
image_files = [f for f in os.listdir(folder_path)
if f.lower().endswith(('.jpg', '.jpeg', '.png', '.bmp'))]
features = {}
similarity_matrix = []
# 提取所有图片特征
for img_file in image_files:
img_path = os.path.join(folder_path, img_file)
descriptors = self.extract_features(img_path)
features[img_file] = descriptors
# 计算相似度矩阵
for i, img1 in enumerate(image_files):
similarities = []
for j, img2 in enumerate(image_files):
if i == j:
similarities.append(1.0)
elif j < i:
similarities.append(similarity_matrix[j][i])
else:
sim = self.calculate_similarity(
features[img1], features[img2]
)
similarities.append(sim)
similarity_matrix.append(similarities)
# 使用DBSCAN聚类
clustering = DBSCAN(eps=similarity_threshold, min_samples=2,
metric='precomputed')
distance_matrix = 1 - np.array(similarity_matrix)
labels = clustering.fit_predict(distance_matrix)
# 组织结果
similar_groups = {}
for idx, label in enumerate(labels):
if label != -1: # -1表示噪声点（无相似图片）
if label not in similar_groups:
similar_groups[label] = []
similar_groups[label].append(image_files[idx])
return similar_groups
# 使用示例
finder = ImageSimilarityFinder()
similar_groups = finder.find_similar_groups("your_image_folder", similarity_threshold=0.3)
for group_id, images in similar_groups.items():
print(f"相似组 {group_id} ({len(images)}张):")
for img in images:
print(f" - {img}")
使用直方图比较（快速但粗略）
import cv2
import os
from scipy.spatial.distance import cosine
from collections import defaultdict
def compare_histograms(image_folder, threshold=0.1):
"""
使用直方图比较查找相似图片
Args:
image_folder: 图片文件夹
threshold: 余弦相似度阈值（越小表示越相似）
"""
def get_histogram(image_path):
img = cv2.imread(image_path)
if img is None:
return None
# 计算颜色直方图
hist = cv2.calcHist([img], [0, 1, 2], None, [8, 8, 8],
[0, 256, 0, 256, 0, 256])
# 规范化
cv2.normalize(hist, hist)
return hist.flatten()
image_files = [f for f in os.listdir(image_folder)
if f.lower().endswith(('.jpg', '.jpeg', '.png'))]
histograms = {}
for img_file in image_files:
img_path = os.path.join(image_folder, img_file)
hist = get_histogram(img_path)
if hist is not None:
histograms[img_file] = hist
# 比较所有图片
similar_groups = defaultdict(list)
processed = set()
for i, (img1, hist1) in enumerate(histograms.items()):
if img1 in processed:
continue
group = [img1]
processed.add(img1)
for j, (img2, hist2) in enumerate(histograms.items()):
if img2 in processed or i == j:
continue
similarity = cosine(hist1, hist2)
if similarity < threshold:
group.append(img2)
processed.add(img2)
if len(group) > 1:
similar_groups[f"Group_{i}"] = group
return similar_groups
# 使用示例
similar_groups = compare_histograms("your_image_folder", threshold=0.1)
for group_name, images in similar_groups.items():
print(f"{group_name} ({len(images)}张):")
for img in images:
print(f" - {img}")
完整脚本：去重并保留最佳质量
#!/usr/bin/env python3
import os
import shutil
from PIL import Image
import imagehash
from pathlib import Path
class ImageDeduplicator:
def __init__(self, source_folder, duplicate_folder="duplicates"):
self.source_folder = Path(source_folder)
self.duplicate_folder = Path(duplicate_folder)
self.duplicate_folder.mkdir(exist_ok=True)
def get_image_quality(self, image_path):
"""获取图片质量评分（基于文件大小和分辨率）"""
img = Image.open(image_path)
width, height = img.size
file_size = os.path.getsize(image_path)
# 综合评分：分辨率 * 文件大小
return width * height * file_size
def find_and_remove_duplicates(self, hash_size=8, threshold=5):
"""查找并移除重复图片"""
# 获取所有图片
extensions = {'.jpg', '.jpeg', '.png', '.gif', '.bmp'}
images = [f for f in self.source_folder.iterdir()
if f.suffix.lower() in extensions]
hashes = {}
duplicates_found = 0
for img_path in images:
try:
with Image.open(img_path) as img:
phash = imagehash.phash(img, hash_size=hash_size)
# 检查是否与已存在的哈希值相似
is_duplicate = False
for existing_hash, existing_path in hashes.items():
if phash - existing_hash <= threshold:
# 比较质量，保留更好的
if self.get_image_quality(img_path) > self.get_image_quality(existing_path):
print(f"替换: {existing_path} -> {img_path.name}")
shutil.move(str(existing_path), str(self.duplicate_folder / existing_path.name))
hashes[existing_hash] = img_path
else:
print(f"重复: {img_path.name} (与 {existing_path.name} 相似)")
shutil.move(str(img_path), str(self.duplicate_folder / img_path.name))
is_duplicate = True
duplicates_found += 1
break
if not is_duplicate:
hashes[phash] = img_path
except Exception as e:
print(f"处理 {img_path.name} 时出错: {e}")
print(f"\n总共发现 {duplicates_found} 个重复文件")
print(f"重复文件已移动到: {self.duplicate_folder}")
# 使用示例
if __name__ == "__main__":
deduplicator = ImageDeduplicator("your_photo_folder", "duplicate_photos")
deduplicator.find_and_remove_duplicates(hash_size=8, threshold=5)
安装依赖
# 基础依赖 pip install Pillow imagehash # 如果需要OpenCV版本 pip install opencv-python scikit-learn # 如果需要直方图版本 pip install opencv-python scipy
使用建议
-
选择合适的哈希大小：
hash_size=8：快速但可能漏掉一些相似的图片hash_size=16：更精确但较慢
-
调整阈值：
- 阈值越小,要求越严格（只找非常相似的图片）
- 阈值越大,能找到更多相似但不完全相同的图片
-
性能优化：
- 图片数量多时,建议分批处理
- 可以先缩小图片尺寸再处理
选择哪种方法取决于你的具体需求：
- 快速去重：使用图像哈希方法
- 精确匹配：使用特征匹配方法
- 简单测试：使用直方图方法
