#!/usr/bin/env python3 """Сбор всех страниц пагинированного API.""" from __future__ import annotations import argparse import json from pathlib import Path from urllib.error import HTTPError, URLError from urllib.parse import urlencode, urlsplit, urlunsplit from urllib.request import Request, urlopen def with_query(url: str, values: dict[str, object]) -> str: parts = urlsplit(url) query = dict() if parts.query: for item in parts.query.split("&"): if "=" in item: key, value = item.split("=", 1) query[key] = value query.update({key: str(value) for key, value in values.items()}) return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment)) def fetch(url: str, timeout: float) -> object: request = Request(url, headers={"Accept": "application/json", "User-Agent": "popovdev-scriptdock/1.0"}) with urlopen(request, timeout=timeout) as response: return json.loads(response.read(5_000_000).decode("utf-8")) def find_items(payload: object) -> list: if isinstance(payload, list): return payload if isinstance(payload, dict): for key in ("items", "data", "results", "records"): value = payload.get(key) if isinstance(value, list): return value return [] def main() -> int: parser = argparse.ArgumentParser(description="Собрать все страницы API") parser.add_argument("url", help="Базовый URL коллекции") parser.add_argument("--page-param", default="page") parser.add_argument("--limit-param", default="per_page") parser.add_argument("--per-page", type=int, default=50) parser.add_argument("--start", type=int, default=1, help="Номер первой страницы") parser.add_argument("--step", type=int, default=1, help="Шаг между страницами") parser.add_argument("--max-pages", type=int, default=20) parser.add_argument("--timeout", type=float, default=15.0) parser.add_argument("--output", type=Path, default=Path("pagination.json")) args = parser.parse_args() collected: list = [] seen_ids: set[str] = set() page = args.start for number in range(1, args.max_pages + 1): url = with_query(args.url, {args.page_param: page, args.limit_param: args.per_page}) try: payload = fetch(url, args.timeout) except (HTTPError, URLError, json.JSONDecodeError, UnicodeDecodeError) as error: print(f"Ошибка на странице {page}: {error}") return 2 items = find_items(payload) print(f"Страница {page}: {len(items)} элементов") if not items: break added = 0 for item in items: marker = None if isinstance(item, dict): for key in ("id", "uuid", "slug"): if key in item: marker = str(item[key]) break if marker is not None: if marker in seen_ids: continue seen_ids.add(marker) collected.append(item) added += 1 print(f" добавлено: {added}, всего: {len(collected)}") if len(items) < args.per_page: break page += args.step else: print(f"Достигнут лимит страниц: {args.max_pages}") args.output.write_text(json.dumps(collected, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") print(f"Сохранено: {args.output} ({len(collected)} элементов)") return 0 if __name__ == "__main__": raise SystemExit(main())