#!/usr/bin/env python3 """Проверка локальных и внешних ссылок в Markdown и HTML.""" from __future__ import annotations import argparse import csv import json import re import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor from pathlib import Path from urllib.parse import unquote, urlparse MD_LINK = re.compile(r"\[([^\]]*)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)") HTML_LINK = re.compile(r"(?:href|src)\s*=\s*[\"']([^\"']+)[\"']", re.IGNORECASE) HEADING = re.compile(r"^(#{1,6})\s+(.*)$") HTML_ID = re.compile(r"\b(?:id|name)\s*=\s*[\"']([^\"']+)[\"']", re.IGNORECASE) FENCE = re.compile(r"^\s*(```|~~~)") SKIP_SCHEMES = ("mailto:", "tel:", "javascript:", "data:", "ftp:") def slugify(text: str) -> str: text = re.sub(r"`([^`]*)`", r"\1", text) text = re.sub(r"\[([^\]]+)\]\([^)]+\)", r"\1", text) text = text.strip().lower() text = re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE) return re.sub(r"\s+", "-", text).strip("-") def anchors(path: Path) -> set[str]: try: text = path.read_text(encoding="utf-8", errors="replace") except OSError: return set() if path.suffix.lower() == ".html": return set(HTML_ID.findall(text)) result = set() in_fence = False for line in text.splitlines(): if FENCE.match(line): in_fence = not in_fence continue if in_fence: continue match = HEADING.match(line) if match: result.add(slugify(match.group(2))) return result def line_number(text: str, position: int) -> int: return text.count("\n", 0, position) + 1 def extract(path: Path) -> list[tuple[int, str]]: text = path.read_text(encoding="utf-8", errors="replace") links: list[tuple[int, str]] = [] for match in MD_LINK.finditer(text): links.append((line_number(text, match.start()), match.group(2).strip())) for match in HTML_LINK.finditer(text): links.append((line_number(text, match.start()), match.group(1).strip())) return links def check_http(url: str, timeout: float) -> str | None: headers = {"User-Agent": "PopovDev-LinkChecker/1.0"} for method in ("HEAD", "GET"): request = urllib.request.Request(url, headers=headers, method=method) try: with urllib.request.urlopen(request, timeout=timeout) as response: if response.status < 400: return None return f"HTTP {response.status}" except urllib.error.HTTPError as error: if method == "HEAD" and error.code in (403, 405, 429): continue return f"HTTP {error.code}" except urllib.error.URLError as error: return f"сеть: {error.reason}" except TimeoutError: return "таймаут" except ValueError: return "некорректный URL" return "недоступен" def main() -> int: parser = argparse.ArgumentParser(description="Проверить ссылки в документах") parser.add_argument("--path", type=Path, required=True, help="Файл или папка") parser.add_argument("--glob", default="*", help="Маска файлов") parser.add_argument("--timeout", type=float, default=10.0, help="Таймаут HTTP в секундах") parser.add_argument("--skip-external", action="store_true", help="Не проверять HTTP") parser.add_argument("--csv", type=Path, help="Сохранить отчёт в CSV") parser.add_argument("--json", type=Path, help="Сохранить отчёт в JSON") parser.add_argument("--fail-on-broken", action="store_true", help="Код 1 при битых ссылках") args = parser.parse_args() root = args.path.expanduser().resolve() if not root.exists(): parser.error(f"Путь не найден: {root}") files = [root] if root.is_file() else sorted(path for path in root.rglob(args.glob) if path.is_file()) broken: list[dict[str, object]] = [] external: dict[str, list[tuple[Path, int, str]]] = {} anchor_cache: dict[Path, set[str]] = {} for source in files: for line, link in extract(source): lower = link.lower() if lower.startswith(SKIP_SCHEMES) or link.startswith("#"): if link.startswith("#"): cache_key = source if cache_key not in anchor_cache: anchor_cache[cache_key] = anchors(source) if unquote(link[1:]) not in anchor_cache[cache_key]: broken.append({"file": str(source), "line": line, "link": link, "reason": "якорь не найден"}) continue parsed = urlparse(link) if parsed.scheme in ("http", "https"): if not args.skip_external: external.setdefault(link, []).append((source, line, link)) continue target_part, _, fragment = link.partition("#") target_part = target_part.split("?", 1)[0] target = (source.parent / unquote(target_part)).resolve() if target_part else source if not target.exists(): broken.append({"file": str(source), "line": line, "link": link, "reason": "файл не найден"}) continue if fragment and target.suffix.lower() in (".md", ".html", ".htm"): if target not in anchor_cache: anchor_cache[target] = anchors(target) if unquote(fragment) not in anchor_cache[target]: broken.append({"file": str(source), "line": line, "link": link, "reason": "якорь не найден"}) if external: with ThreadPoolExecutor(max_workers=8) as pool: results = {url: pool.submit(check_http, url, args.timeout) for url in external} for url, places in external.items(): reason = results[url].result() if reason: for source, line, link in places: broken.append({"file": str(source), "line": line, "link": link, "reason": reason}) if not broken: print(f"Проверено файлов: {len(files)}, битых ссылок нет") return 0 for item in broken: print(f"{item['file']}:{item['line']}: {item['link']} - {item['reason']}") print(f"Битых ссылок: {len(broken)}") if args.csv: with args.csv.expanduser().open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=["file", "line", "link", "reason"]) writer.writeheader() writer.writerows(broken) print(f"CSV: {args.csv}") if args.json: args.json.expanduser().write_text( json.dumps({"broken": broken}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) print(f"JSON: {args.json}") return 1 if args.fail_on_broken else 0 if __name__ == "__main__": raise SystemExit(main())