#!/usr/bin/env python3 """Поиск дубликатов внутри zip или tar.""" from __future__ import annotations import argparse import hashlib import json import tarfile import zipfile from collections import defaultdict from pathlib import Path def digest_stream(handle) -> str: digest = hashlib.sha256() while chunk := handle.read(1024 * 1024): digest.update(chunk) return digest.hexdigest() def main() -> int: parser = argparse.ArgumentParser(description="Найти дубликаты внутри архива") parser.add_argument("archive", type=Path, help="Zip или tar архив") parser.add_argument("--min-size", type=int, default=1, help="Минимальный размер файла") parser.add_argument("--json", type=Path, help="Сохранить отчёт в JSON") args = parser.parse_args() path = args.archive.expanduser().resolve() if not path.is_file(): parser.error(f"Архив не найден: {path}") entries: list[tuple[str, int, str]] = [] if zipfile.is_zipfile(path): kind = "zip" with zipfile.ZipFile(path) as archive: for info in archive.infolist(): if info.is_dir() or info.file_size < args.min_size: continue entries.append((info.filename, info.file_size, "")) by_size = defaultdict(list) for name, size, _ in entries: by_size[size].append(name) for size, names in by_size.items(): if len(names) < 2: continue for name in names: try: with archive.open(name) as handle: digest = digest_stream(handle) except (RuntimeError, zipfile.BadZipFile): continue for index, item in enumerate(entries): if item[0] == name: entries[index] = (name, size, digest) break elif tarfile.is_tarfile(path): kind = "tar" with tarfile.open(path, mode="r:*") as archive: for member in archive: if not member.isfile() or member.size < args.min_size: continue entries.append((member.name, member.size, "")) by_size = defaultdict(list) for name, size, _ in entries: by_size[size].append(name) for size, names in by_size.items(): if len(names) < 2: continue for name in names: source = archive.extractfile(name) if source is None: continue with source: digest = digest_stream(source) for index, item in enumerate(entries): if item[0] == name: entries[index] = (name, size, digest) break else: print("Формат не распознан. Поддерживаются zip и tar.") return 2 by_hash = defaultdict(list) for name, size, digest in entries: if digest: by_hash[(size, digest)].append(name) groups = [ {"size": size, "sha256": digest, "files": names} for (size, digest), names in by_hash.items() if len(names) > 1 ] groups.sort(key=lambda item: item["size"] * (len(item["files"]) - 1), reverse=True) wasted = sum(item["size"] * (len(item["files"]) - 1) for item in groups) for number, group in enumerate(groups, 1): print(f"Группа {number}: {len(group['files'])} файлов по {group['size'] / 1024:.1f} КБ") for name in group["files"]: print(f" {name}") if not groups: print("Дубликаты не найдены.") else: print(f"Групп: {len(groups)}, потенциально лишних байт: {wasted}") if args.json: report = {"archive": str(path), "format": kind, "groups": groups, "wasted": wasted} args.json.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") return 0 if __name__ == "__main__": raise SystemExit(main())