#!/usr/bin/env python3
"""Преобразование HTML в чистый текст."""
from __future__ import annotations
import argparse
import csv
import re
from pathlib import Path
from bs4 import BeautifulSoup
BLOCK_TAGS = ["p", "div", "section", "article", "header", "footer", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6"]
def clean_html(source: Path, encoding: str) -> tuple[str, list[tuple[str, str]]]:
html = source.read_text(encoding=encoding, errors="replace")
soup = BeautifulSoup(html, "html.parser")
for tag in soup(["script", "style", "noscript", "template", "svg", "head", "title", "meta", "link"]):
tag.decompose()
links = [(anchor.get_text(" ", strip=True), anchor.get("href", "")) for anchor in soup.find_all("a", href=True)]
for br in soup.find_all("br"):
br.replace_with("\n")
for tag in soup.find_all(BLOCK_TAGS):
tag.append("\n")
text = soup.get_text("")
lines = [re.sub(r"[ \t]+", " ", line).strip() for line in text.splitlines()]
cleaned: list[str] = []
for line in lines:
if line:
cleaned.append(line)
elif cleaned and cleaned[-1] != "":
cleaned.append("")
return "\n".join(cleaned).strip() + "\n", links
def main() -> int:
parser = argparse.ArgumentParser(description="Очистить HTML до текста")
parser.add_argument("--path", type=Path, required=True, help="Файл или папка")
parser.add_argument("--glob", default="*.html", help="Маска файлов")
parser.add_argument("--encoding", default="utf-8", help="Кодировка входа")
parser.add_argument("--output-dir", type=Path, default=Path("text-out"), help="Папка для текста")
parser.add_argument("--links", type=Path, help="CSV со ссылками")
parser.add_argument("--force", action="store_true", help="Перезаписать результаты")
args = parser.parse_args()
root = args.path.expanduser().resolve()
if not root.exists():
parser.error(f"Путь не найден: {root}")
out_dir = args.output_dir.expanduser().resolve()
files = [root] if root.is_file() else sorted(path for path in root.rglob(args.glob) if path.is_file())
all_links: list[tuple[str, str, str]] = []
written = 0
for source in files:
text, links = clean_html(source, args.encoding)
relative = source.relative_to(root) if root.is_dir() else Path(source.name)
target = out_dir / relative.with_suffix(".txt")
if target.resolve() == source.resolve():
print(f"Пропуск, выходной путь совпадает с входным: {source}")
continue
if target.exists() and not args.force:
print(f"Пропуск, файл уже есть: {target}")
continue
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(text, encoding="utf-8")
written += 1
for label, href in links:
all_links.append((str(source), label, href))
print(f"Сохранено: {target}")
if args.links and all_links:
with args.links.expanduser().open("w", encoding="utf-8", newline="") as handle:
writer = csv.writer(handle)
writer.writerow(["source", "text", "href"])
writer.writerows(all_links)
print(f"Ссылки: {args.links}")
print(f"Файлов обработано: {written}")
return 0
if __name__ == "__main__":
raise SystemExit(main())