Python 实现 查找相同的图像文件flyfish找相同的图像文件importosfromPILimportImageimportimagehash# 修改配置 # 要扫描的根文件夹支持多层子目录替换自己的路径ROOT_FOLDERr/media/classes/# 结果输出的文本文件路径OUTPUT_TXT重复图片结果.txt# 汉明距离阈值0像素几乎完全一致2近似重复数值越大判定越宽松# 视频抽帧去重推荐 2仅找完全相同的文件设为 0HASH_THRESHOLD0# 支持扫描的图片格式SUPPORTED_FORMATS(.jpg,.jpeg,.png,.bmp,.webp,.tiff)# defget_all_image_paths(root_dir):递归遍历所有子文件夹收集所有图片的绝对路径image_list[]fordirpath,_,filenamesinos.walk(root_dir):forfilenameinfilenames:iffilename.lower().endswith(SUPPORTED_FORMATS):abs_pathos.path.abspath(os.path.join(dirpath,filename))image_list.append(abs_path)returnimage_listdefcalculate_dhash(img_path):计算单张图片的dHash指纹读取失败返回Nonetry:withImage.open(img_path)asimg:returnimagehash.dhash(img)exceptExceptionase:print(f[跳过异常文件]{img_path}| 原因:{str(e)})returnNonedeffind_duplicate_groups(image_paths):将重复图片分组返回重复组列表每组包含所有互为重复的文件路径# 每个元素格式(代表哈希值, [该组所有图片路径])groups[]totallen(image_paths)foridx,img_pathinenumerate(image_paths,1):# 打印进度print(f处理进度:{idx}/{total}-{os.path.basename(img_path)})img_hashcalculate_dhash(img_path)ifimg_hashisNone:continue# 遍历已有分组查找匹配的组matchedFalseforgroup_hash,group_filesingroups:ifimg_hash-group_hashHASH_THRESHOLD:group_files.append(img_path)matchedTruebreak# 没有匹配的组新建一个组ifnotmatched:groups.append((img_hash,[img_path]))# 只保留文件数1的重复组duplicate_groups[filesfor_,filesingroupsiflen(files)1]returnduplicate_groupsdefwrite_result_to_txt(duplicate_groups,output_path):将重复结果写入文本文件按组展示绝对路径withopen(output_path,w,encodingutf-8)asf:f.write(f扫描根目录:{os.path.abspath(ROOT_FOLDER)}\n)f.write(f汉明距离阈值:{HASH_THRESHOLD}\n)f.write(f共找到{len(duplicate_groups)}组重复图片\n)f.write(*90\n\n)forgroup_id,filesinenumerate(duplicate_groups,1):f.write(f第{group_id}组重复共{len(files)}张\n)forfile_pathinfiles:f.write(f{file_path}\n)f.write(\n-*90\n\n)print(f\n处理完成结果已写入:{os.path.abspath(output_path)})if__name____main__:# 1. 收集所有图片路径print(正在扫描文件夹...)all_imagesget_all_image_paths(ROOT_FOLDER)print(f共找到{len(all_images)}张图片开始计算哈希并查找重复...\n)# 2. 查找重复分组dup_groupsfind_duplicate_groups(all_images)# 3. 写入结果文件ifdup_groups:write_result_to_txt(dup_groups,OUTPUT_TXT)else:print(\n扫描完成未找到重复图片。)检出重复图片后每组保留首张留在原目录其余按原文件夹层级移到备份目录importosimportshutilfromPILimportImageimportimagehash# 配置区 # 原始图片根目录会递归扫描所有子文件夹ROOT_FOLDERr/media/classes/# 重复图片移动到的目标目录自动保留原始目录结构MOVE_FOLDERr./duplicate_images_backup# 结果清单保存路径RESULT_TXT重复图片结果.txt# 汉明距离阈值0完全相同数值越大判定越宽松HASH_THRESHOLD0# 支持的图片格式SUPPORTED_FORMATS(.jpg,.jpeg,.png,.bmp,.webp,.tiff)# defget_all_images(root_dir):递归获取所有图片绝对路径img_list[]fordirpath,_,filenamesinos.walk(root_dir):fornameinfilenames:ifname.lower().endswith(SUPPORTED_FORMATS):img_list.append(os.path.abspath(os.path.join(dirpath,name)))returnimg_listdefcompute_hash(img_path):计算dHash指纹失败返回Nonetry:withImage.open(img_path)asimg:returnimagehash.dhash(img)exceptExceptionase:print(f[跳过]{img_path}| 错误:{e})returnNonedeffind_duplicates(img_paths):分组重复图片返回重复组列表groups[]totallen(img_paths)fori,pathinenumerate(img_paths,1):print(f处理中:{i}/{total}-{os.path.basename(path)})img_hashcompute_hash(path)ifimg_hashisNone:continue# 查找匹配组matchedFalseforgroup_hash,group_filesingroups:ifimg_hash-group_hashHASH_THRESHOLD:group_files.append(path)matchedTruebreak# 新建组ifnotmatched:groups.append((img_hash,[path]))# 只保留有重复的组return[filesfor_,filesingroupsiflen(files)1]defmove_duplicates(dup_groups,root_dir,move_dir):每组保留第一张其余按原目录结构移动到目标文件夹root_absos.path.abspath(root_dir)move_absos.path.abspath(move_dir)count0forgid,filesinenumerate(dup_groups,1):keepfiles[0]move_listfiles[1:]print(f\n第{gid}组保留{os.path.basename(keep)}移动{len(move_list)}张)forsrcinmove_list:try:# 计算相对路径保持目录结构rel_pathos.path.relpath(src,root_abs)destos.path.join(move_abs,rel_path)os.makedirs(os.path.dirname(dest),exist_okTrue)# 同名文件自动加序号避免覆盖ifos.path.exists(dest):base,extos.path.splitext(dest)n1whileos.path.exists(f{base}_{n}{ext}):n1destf{base}_{n}{ext}shutil.move(src,dest)print(f ✓{rel_path})count1exceptExceptionase:print(f ✗ 移动失败{src}|{e})returncountdefsave_result(dup_groups,output_path):保存重复结果到文本文件withopen(output_path,w,encodingutf-8)asf:f.write(f扫描目录:{os.path.abspath(ROOT_FOLDER)}\n)f.write(f汉明阈值:{HASH_THRESHOLD}\n)f.write(f重复组数:{len(dup_groups)}\n)f.write(*80\n\n)forgid,filesinenumerate(dup_groups,1):f.write(f第{gid}组共{len(files)}张首张保留\n)fori,pinenumerate(files):f.write(f{[保留]ifi0else[已移动]}{p}\n)f.write(\n-*80\n\n)print(f\n结果清单已保存:{os.path.abspath(output_path)})if__name____main__:print(正在扫描图片...)all_imgsget_all_images(ROOT_FOLDER)print(f找到{len(all_imgs)}张图片开始去重检测...\n)dup_groupsfind_duplicates(all_imgs)ifnotdup_groups:print(\n未发现重复图片。)else:save_result(dup_groups,RESULT_TXT)print(\n开始移动重复文件...)total_movedmove_duplicates(dup_groups,ROOT_FOLDER,MOVE_FOLDER)print(f\n 处理完成 )print(f共移动重复图片:{total_moved}张)print(f备份目录:{os.path.abspath(MOVE_FOLDER)})
Python 实现 查找相同的图像文件flyfish找相同的图像文件importosfromPILimportImageimportimagehash# 修改配置 # 要扫描的根文件夹支持多层子目录替换自己的路径ROOT_FOLDERr/media/classes/# 结果输出的文本文件路径OUTPUT_TXT重复图片结果.txt# 汉明距离阈值0像素几乎完全一致2近似重复数值越大判定越宽松# 视频抽帧去重推荐 2仅找完全相同的文件设为 0HASH_THRESHOLD0# 支持扫描的图片格式SUPPORTED_FORMATS(.jpg,.jpeg,.png,.bmp,.webp,.tiff)# defget_all_image_paths(root_dir):递归遍历所有子文件夹收集所有图片的绝对路径image_list[]fordirpath,_,filenamesinos.walk(root_dir):forfilenameinfilenames:iffilename.lower().endswith(SUPPORTED_FORMATS):abs_pathos.path.abspath(os.path.join(dirpath,filename))image_list.append(abs_path)returnimage_listdefcalculate_dhash(img_path):计算单张图片的dHash指纹读取失败返回Nonetry:withImage.open(img_path)asimg:returnimagehash.dhash(img)exceptExceptionase:print(f[跳过异常文件]{img_path}| 原因:{str(e)})returnNonedeffind_duplicate_groups(image_paths):将重复图片分组返回重复组列表每组包含所有互为重复的文件路径# 每个元素格式(代表哈希值, [该组所有图片路径])groups[]totallen(image_paths)foridx,img_pathinenumerate(image_paths,1):# 打印进度print(f处理进度:{idx}/{total}-{os.path.basename(img_path)})img_hashcalculate_dhash(img_path)ifimg_hashisNone:continue# 遍历已有分组查找匹配的组matchedFalseforgroup_hash,group_filesingroups:ifimg_hash-group_hashHASH_THRESHOLD:group_files.append(img_path)matchedTruebreak# 没有匹配的组新建一个组ifnotmatched:groups.append((img_hash,[img_path]))# 只保留文件数1的重复组duplicate_groups[filesfor_,filesingroupsiflen(files)1]returnduplicate_groupsdefwrite_result_to_txt(duplicate_groups,output_path):将重复结果写入文本文件按组展示绝对路径withopen(output_path,w,encodingutf-8)asf:f.write(f扫描根目录:{os.path.abspath(ROOT_FOLDER)}\n)f.write(f汉明距离阈值:{HASH_THRESHOLD}\n)f.write(f共找到{len(duplicate_groups)}组重复图片\n)f.write(*90\n\n)forgroup_id,filesinenumerate(duplicate_groups,1):f.write(f第{group_id}组重复共{len(files)}张\n)forfile_pathinfiles:f.write(f{file_path}\n)f.write(\n-*90\n\n)print(f\n处理完成结果已写入:{os.path.abspath(output_path)})if__name____main__:# 1. 收集所有图片路径print(正在扫描文件夹...)all_imagesget_all_image_paths(ROOT_FOLDER)print(f共找到{len(all_images)}张图片开始计算哈希并查找重复...\n)# 2. 查找重复分组dup_groupsfind_duplicate_groups(all_images)# 3. 写入结果文件ifdup_groups:write_result_to_txt(dup_groups,OUTPUT_TXT)else:print(\n扫描完成未找到重复图片。)检出重复图片后每组保留首张留在原目录其余按原文件夹层级移到备份目录importosimportshutilfromPILimportImageimportimagehash# 配置区 # 原始图片根目录会递归扫描所有子文件夹ROOT_FOLDERr/media/classes/# 重复图片移动到的目标目录自动保留原始目录结构MOVE_FOLDERr./duplicate_images_backup# 结果清单保存路径RESULT_TXT重复图片结果.txt# 汉明距离阈值0完全相同数值越大判定越宽松HASH_THRESHOLD0# 支持的图片格式SUPPORTED_FORMATS(.jpg,.jpeg,.png,.bmp,.webp,.tiff)# defget_all_images(root_dir):递归获取所有图片绝对路径img_list[]fordirpath,_,filenamesinos.walk(root_dir):fornameinfilenames:ifname.lower().endswith(SUPPORTED_FORMATS):img_list.append(os.path.abspath(os.path.join(dirpath,name)))returnimg_listdefcompute_hash(img_path):计算dHash指纹失败返回Nonetry:withImage.open(img_path)asimg:returnimagehash.dhash(img)exceptExceptionase:print(f[跳过]{img_path}| 错误:{e})returnNonedeffind_duplicates(img_paths):分组重复图片返回重复组列表groups[]totallen(img_paths)fori,pathinenumerate(img_paths,1):print(f处理中:{i}/{total}-{os.path.basename(path)})img_hashcompute_hash(path)ifimg_hashisNone:continue# 查找匹配组matchedFalseforgroup_hash,group_filesingroups:ifimg_hash-group_hashHASH_THRESHOLD:group_files.append(path)matchedTruebreak# 新建组ifnotmatched:groups.append((img_hash,[path]))# 只保留有重复的组return[filesfor_,filesingroupsiflen(files)1]defmove_duplicates(dup_groups,root_dir,move_dir):每组保留第一张其余按原目录结构移动到目标文件夹root_absos.path.abspath(root_dir)move_absos.path.abspath(move_dir)count0forgid,filesinenumerate(dup_groups,1):keepfiles[0]move_listfiles[1:]print(f\n第{gid}组保留{os.path.basename(keep)}移动{len(move_list)}张)forsrcinmove_list:try:# 计算相对路径保持目录结构rel_pathos.path.relpath(src,root_abs)destos.path.join(move_abs,rel_path)os.makedirs(os.path.dirname(dest),exist_okTrue)# 同名文件自动加序号避免覆盖ifos.path.exists(dest):base,extos.path.splitext(dest)n1whileos.path.exists(f{base}_{n}{ext}):n1destf{base}_{n}{ext}shutil.move(src,dest)print(f ✓{rel_path})count1exceptExceptionase:print(f ✗ 移动失败{src}|{e})returncountdefsave_result(dup_groups,output_path):保存重复结果到文本文件withopen(output_path,w,encodingutf-8)asf:f.write(f扫描目录:{os.path.abspath(ROOT_FOLDER)}\n)f.write(f汉明阈值:{HASH_THRESHOLD}\n)f.write(f重复组数:{len(dup_groups)}\n)f.write(*80\n\n)forgid,filesinenumerate(dup_groups,1):f.write(f第{gid}组共{len(files)}张首张保留\n)fori,pinenumerate(files):f.write(f{[保留]ifi0else[已移动]}{p}\n)f.write(\n-*80\n\n)print(f\n结果清单已保存:{os.path.abspath(output_path)})if__name____main__:print(正在扫描图片...)all_imgsget_all_images(ROOT_FOLDER)print(f找到{len(all_imgs)}张图片开始去重检测...\n)dup_groupsfind_duplicates(all_imgs)ifnotdup_groups:print(\n未发现重复图片。)else:save_result(dup_groups,RESULT_TXT)print(\n开始移动重复文件...)total_movedmove_duplicates(dup_groups,ROOT_FOLDER,MOVE_FOLDER)print(f\n 处理完成 )print(f共移动重复图片:{total_moved}张)print(f备份目录:{os.path.abspath(MOVE_FOLDER)})
