#!/usr/bin/env python3 """Поиск дубликатов файлов по размеру и SHA-256.""" from __future__ import annotations import argparse import hashlib import json import os from collections import defaultdict from pathlib import Path DEFAULT_EXCLUDES = {".git", ".hg", ".svn", "node_modules", "__pycache__", ".venv", "venv"} def human_size(size: int) -> str: value = float(size) for unit in ("B", "KB", "MB", "GB", "TB"): if value < 1024 or unit == "TB": return f"{int(value)} B" if unit == "B" else f"{value:.1f} {unit}" value /= 1024 return f"{size} B" def file_hash(path: Path, chunk_size: int = 1024 * 1024) -> str: digest = hashlib.sha256() with path.open("rb") as handle: while chunk := handle.read(chunk_size): digest.update(chunk) return digest.hexdigest() def iter_files(root: Path, excludes: set[str]): for current, dirs, files in os.walk(root): current_path = Path(current) dirs[:] = [ name for name in dirs if name not in excludes and not (current_path / name).is_symlink() ] for name in files: path = current_path / name if path.is_symlink() or not path.is_file(): continue yield path def find_duplicates(root: Path, min_size: int, excludes: set[str]): by_size: dict[int, list[Path]] = defaultdict(list) for path in iter_files(root, excludes): try: size = path.stat().st_size except OSError as error: print(f"Пропуск {path}: {error}") continue if size >= min_size: by_size[size].append(path) groups = [] for size, paths in by_size.items(): if len(paths) < 2: continue by_hash: dict[str, list[Path]] = defaultdict(list) for path in paths: try: by_hash[file_hash(path)].append(path) except OSError as error: print(f"Не удалось прочитать {path}: {error}") for digest, same in by_hash.items(): if len(same) > 1: groups.append( { "size": size, "sha256": digest, "wasted": size * (len(same) - 1), "files": [str(path.relative_to(root)) for path in sorted(same)], } ) groups.sort(key=lambda item: item["wasted"], reverse=True) return groups def main() -> int: parser = argparse.ArgumentParser(description="Найти дубликаты файлов") parser.add_argument("--path", type=Path, default=Path("."), help="Папка для поиска") parser.add_argument("--min-size", type=int, default=1, help="Минимальный размер файла в байтах") parser.add_argument("--exclude", action="append", default=[], help="Имя папки, которую нужно пропустить") parser.add_argument("--json", type=Path, help="Сохранить отчёт в JSON") args = parser.parse_args() root = args.path.expanduser().resolve() if not root.is_dir(): parser.error(f"Папка не найдена: {root}") if args.min_size < 0: parser.error("--min-size не может быть отрицательным") excludes = DEFAULT_EXCLUDES | {item.strip() for item in args.exclude if item.strip()} groups = find_duplicates(root, args.min_size, excludes) if not groups: print("Дубликаты не найдены.") return 0 total_wasted = sum(item["wasted"] for item in groups) for index, group in enumerate(groups, start=1): print(f"\nГруппа {index}: {human_size(group['size'])} на файл, копий: {len(group['files'])}") print(f"SHA-256: {group['sha256']}") for relative in group["files"]: print(f" {relative}") print(f"\nГрупп: {len(groups)}, можно освободить: {human_size(total_wasted)}") if args.json: output = args.json.expanduser() output.parent.mkdir(parents=True, exist_ok=True) output.write_text( json.dumps( {"root": str(root), "groups": groups, "wasted": total_wasted}, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) print(f"Отчёт сохранён: {output}") return 0 if __name__ == "__main__": raise SystemExit(main())