#!/usr/bin/env python3 """Поиск повторяющихся абзацев в текстовых файлах.""" from __future__ import annotations import argparse import json import re from collections import defaultdict from pathlib import Path def paragraphs(path: Path, encoding: str) -> list[tuple[int, str]]: lines = path.read_text(encoding=encoding, errors="replace").splitlines() result: list[tuple[int, str]] = [] current: list[str] = [] start = 0 for number, line in enumerate(lines, 1): if line.strip(): if not current: start = number current.append(line) elif current: result.append((start, "\n".join(current))) current = [] if current: result.append((start, "\n".join(current))) return result def normalize(text: str) -> str: return re.sub(r"\s+", " ", text).strip().casefold() def main() -> int: parser = argparse.ArgumentParser(description="Найти повторяющиеся абзацы") parser.add_argument("--path", type=Path, required=True, help="Файл или папка") parser.add_argument("--glob", default="*", help="Маска файлов") parser.add_argument("--encoding", default="utf-8", help="Кодировка") parser.add_argument("--min-length", type=int, default=60, help="Минимальная длина абзаца") parser.add_argument("--json", type=Path, help="Сохранить отчёт в JSON") args = parser.parse_args() root = args.path.expanduser().resolve() if not root.exists(): parser.error(f"Путь не найден: {root}") files = [root] if root.is_file() else sorted(path for path in root.rglob(args.glob) if path.is_file()) index: dict[str, list[dict[str, object]]] = defaultdict(list) originals: dict[str, str] = {} for source in files: for line, text in paragraphs(source, args.encoding): key = normalize(text) if len(key) < args.min_length: continue index[key].append({"file": str(source), "line": line}) originals.setdefault(key, text.strip()) groups = [] for key, places in index.items(): if len(places) > 1: groups.append({"text": originals[key], "count": len(places), "places": places}) groups.sort(key=lambda item: (-item["count"], item["text"])) if not groups: print("Повторяющихся абзацев не найдено") return 0 for group in groups: print(f"\nПовторов: {group['count']}") preview = group["text"][:160].replace("\n", " ") print(f" {preview}") for place in group["places"]: print(f" - {place['file']}:{place['line']}") print(f"\nГрупп повторов: {len(groups)}") if args.json: args.json.expanduser().write_text( json.dumps({"groups": groups}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print(f"JSON: {args.json}") return 0 if __name__ == "__main__": raise SystemExit(main())