#!/usr/bin/env python3 """Небольшой и безопасный загрузчик сайта.""" from __future__ import annotations import argparse import hashlib from html.parser import HTMLParser from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urljoin, urlsplit, urlunsplit from urllib.request import Request, urlopen DEFAULT_HOSTS = {"localhost", "127.0.0.1", "example.com"} class LinkParser(HTMLParser): def __init__(self) -> None: super().__init__() self.links: list[str] = [] def handle_starttag(self, tag: str, attrs) -> None: if tag not in {"a", "link", "script", "img", "source"}: return for name, value in attrs: if name in {"href", "src"} and value: self.links.append(value) def target_path(root: Path, url: str) -> Path: parsed = urlsplit(url) path = parsed.path or "/" parts = [part for part in path.strip("/").split("/") if part] if not parts or path.endswith("/"): parts.append("index.html") name = parts[-1] if "." not in name: parts[-1] = name + ".html" if parsed.query: digest = hashlib.sha256(parsed.query.encode("utf-8")).hexdigest()[:8] suffix = Path(parts[-1]).suffix parts[-1] = f"{Path(parts[-1]).stem}-{digest}{suffix}" result = root.joinpath(*parts) resolved = result.resolve() if root.resolve() not in resolved.parents and resolved != root.resolve(): raise ValueError("путь выходит за пределы каталога") return result def main() -> int: parser = argparse.ArgumentParser(description="Скачать небольшой сайт") parser.add_argument("url", nargs="?", default="http://127.0.0.1:8000/") parser.add_argument("--dir", type=Path, default=Path("site-copy")) parser.add_argument("--max-pages", type=int, default=30, help="Максимум обработанных файлов") parser.add_argument("--max-bytes", type=int, default=5_000_000) parser.add_argument("--timeout", type=float, default=15.0) parser.add_argument("--allow-host", action="append", default=[], help="Дополнительный хост") parser.add_argument("--overwrite", action="store_true", help="Перезаписывать существующие файлы") args = parser.parse_args() allowed = DEFAULT_HOSTS | {host.lower() for host in args.allow_host} root = args.dir root.mkdir(parents=True, exist_ok=True) queue = [args.url] seen: set[str] = set() saved_paths: set[Path] = set() saved = 0 skipped = 0 processed = 0 while queue and processed < args.max_pages: raw = queue.pop(0) parts = urlsplit(raw) if parts.scheme not in {"http", "https"}: continue if (parts.hostname or "").lower() not in allowed: print(f"Пропуск чужого хоста: {raw}") continue url = urlunsplit(parts._replace(fragment="")) if url in seen: continue seen.add(url) request = Request(url, headers={"User-Agent": "popovdev-scriptdock/1.0", "Accept": "*/*"}) try: with urlopen(request, timeout=args.timeout) as response: content_type = response.headers.get("Content-Type", "") data = response.read(args.max_bytes + 1) except (HTTPError, URLError, ValueError) as error: print(f"Ошибка {url}: {error}") continue if len(data) > args.max_bytes: print(f"Пропуск большого файла: {url}") continue try: path = target_path(root, url) except ValueError as error: print(f"Пропуск {url}: {error}") continue if path in saved_paths: print(f"Пропуск повтора пути: {path}") skipped += 1 elif path.exists() and not args.overwrite: print(f"Пропуск существующего файла: {path}") skipped += 1 else: saved_paths.add(path) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) saved += 1 print(f"Сохранено: {path} ({len(data)} байт)") processed += 1 if "html" in content_type.lower(): page = LinkParser() page.feed(data.decode("utf-8", errors="replace")) for link in page.links: child = urlsplit(urljoin(url, link)) if child.scheme in {"http", "https"} and (child.hostname or "").lower() in allowed: queue.append(urlunsplit(child._replace(fragment=""))) print(f"Готово: сохранено {saved}, пропущено {skipped}, обработано {processed}") return 0 if processed else 1 if __name__ == "__main__": raise SystemExit(main())