#!/usr/bin/env python3 """Преобразование HTML в чистый текст.""" from __future__ import annotations import argparse import csv import re from pathlib import Path from bs4 import BeautifulSoup BLOCK_TAGS = ["p", "div", "section", "article", "header", "footer", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6"] def clean_html(source: Path, encoding: str) -> tuple[str, list[tuple[str, str]]]: html = source.read_text(encoding=encoding, errors="replace") soup = BeautifulSoup(html, "html.parser") for tag in soup(["script", "style", "noscript", "template", "svg", "head", "title", "meta", "link"]): tag.decompose() links = [(anchor.get_text(" ", strip=True), anchor.get("href", "")) for anchor in soup.find_all("a", href=True)] for br in soup.find_all("br"): br.replace_with("\n") for tag in soup.find_all(BLOCK_TAGS): tag.append("\n") text = soup.get_text("") lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()] cleaned: list[str] = [] for line in lines: if line: cleaned.append(line) elif cleaned and cleaned[-1] != "": cleaned.append("") return "\n".join(cleaned).strip() + "\n", links def main() -> int: parser = argparse.ArgumentParser(description="Очистить HTML до текста") parser.add_argument("--path", type=Path, required=True, help="Файл или папка") parser.add_argument("--glob", default="*.html", help="Маска файлов") parser.add_argument("--encoding", default="utf-8", help="Кодировка входа") parser.add_argument("--output-dir", type=Path, default=Path("text-out"), help="Папка для текста") parser.add_argument("--links", type=Path, help="CSV со ссылками") parser.add_argument("--force", action="store_true", help="Перезаписать результаты") args = parser.parse_args() root = args.path.expanduser().resolve() if not root.exists(): parser.error(f"Путь не найден: {root}") out_dir = args.output_dir.expanduser().resolve() files = [root] if root.is_file() else sorted(path for path in root.rglob(args.glob) if path.is_file()) all_links: list[tuple[str, str, str]] = [] written = 0 for source in files: text, links = clean_html(source, args.encoding) relative = source.relative_to(root) if root.is_dir() else Path(source.name) target = out_dir / relative.with_suffix(".txt") if target.resolve() == source.resolve(): print(f"Пропуск, выходной путь совпадает с входным: {source}") continue if target.exists() and not args.force: print(f"Пропуск, файл уже есть: {target}") continue target.parent.mkdir(parents=True, exist_ok=True) target.write_text(text, encoding="utf-8") written += 1 for label, href in links: all_links.append((str(source), label, href)) print(f"Сохранено: {target}") if args.links and all_links: with args.links.expanduser().open("w", encoding="utf-8", newline="") as handle: writer = csv.writer(handle) writer.writerow(["source", "text", "href"]) writer.writerows(all_links) print(f"Ссылки: {args.links}") print(f"Файлов обработано: {written}") return 0 if __name__ == "__main__": raise SystemExit(main())