From 4e1211feb64e67511743b8facba912cba272c689 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Sat, 1 Aug 2026 12:26:03 -0400 Subject: [PATCH] BGG API client: disk cache, rate limiting, 202-queue retry, XML parsing httpx client with injectable clock/sleep/rng for testability. Successful responses cached under data/bgg_cache/ keyed by endpoint+params; 202 retries follow the spec schedule (2/5/10/30s, give up after 5); 429/503 get jittered exponential backoff; consecutive requests are spaced rate_limit_seconds apart. Parsers (via defusedxml, per security hook) cover search, thing (+stats/+versions), and collection, including the Not Ranked and error-document cases. Co-Authored-By: Claude Fable 5 --- pyproject.toml | 1 + src/bggpipe/bgg_client.py | 145 +++++++++++++++++++++++++++++++++++ src/bggpipe/models.py | 154 ++++++++++++++++++++++++++++++++++++++ tests/test_client.py | 110 +++++++++++++++++++++++++++ tests/test_models.py | 114 ++++++++++++++++++++++++++++ uv.lock | 11 +++ 6 files changed, 535 insertions(+) create mode 100644 src/bggpipe/bgg_client.py create mode 100644 src/bggpipe/models.py create mode 100644 tests/test_client.py create mode 100644 tests/test_models.py diff --git a/pyproject.toml b/pyproject.toml index 2e84607..4832197 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ dependencies = [ "typer>=0.12", "httpx>=0.27", "rapidfuzz>=3.9", + "defusedxml>=0.7.1", ] [project.scripts] diff --git a/src/bggpipe/bgg_client.py b/src/bggpipe/bgg_client.py new file mode 100644 index 0000000..40a5f38 --- /dev/null +++ b/src/bggpipe/bgg_client.py @@ -0,0 +1,145 @@ +"""BGG XML API2 client: disk cache, rate limiting, 202-queue and 429/503 retry. + +Every successful response is cached under data/bgg_cache/ keyed by +endpoint + params, so re-runs never re-hit the API. The rate limiter +guarantees ≤1 request every rate_limit_seconds to any BGG endpoint. +""" + +from __future__ import annotations + +import hashlib +import random +import re +import time +from collections.abc import Callable, Iterable +from pathlib import Path +from urllib.parse import urlencode + +import httpx + +from bggpipe.models import ( + CollectionItem, + SearchResult, + ThingDetails, + parse_collection, + parse_search, + parse_things, +) + +BASE_URL = "https://boardgamegeek.com/xmlapi2" +QUEUE_BACKOFF = (2.0, 5.0, 10.0, 30.0) # sleeps between the 5 attempts (spec) +MAX_ATTEMPTS = 5 +_UNSAFE = re.compile(r"[^A-Za-z0-9._=,-]+") + + +class BGGQueueTimeout(Exception): + """BGG kept answering 202 (or throttling) past the retry budget.""" + + +def cache_key(endpoint: str, params: dict[str, str]) -> str: + query = urlencode(sorted(params.items())) + digest = hashlib.md5(f"{endpoint}?{query}".encode()).hexdigest()[:10] + slug = _UNSAFE.sub("-", query)[:80].strip("-") + return f"{endpoint}_{slug}_{digest}.xml" + + +class BGGClient: + def __init__( + self, + cache_dir: Path, + rate_limit_seconds: float = 2.0, + transport: httpx.BaseTransport | None = None, + sleep: Callable[[float], None] = time.sleep, + monotonic: Callable[[], float] = time.monotonic, + rng: random.Random | None = None, + ) -> None: + self.cache_dir = cache_dir + self._rate = rate_limit_seconds + self._sleep = sleep + self._monotonic = monotonic + self._rng = rng or random.Random() + self._last_request: float | None = None + self._http = httpx.Client( + base_url=BASE_URL, + timeout=30.0, + headers={"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"}, + transport=transport, + ) + + def _throttle(self) -> None: + if self._last_request is not None: + wait = self._rate - (self._monotonic() - self._last_request) + if wait > 0: + self._sleep(wait) + + def get_xml(self, endpoint: str, params: dict[str, str]) -> str: + """Fetch one endpoint, serving from and filling the disk cache.""" + cache_path = self.cache_dir / cache_key(endpoint, params) + if cache_path.exists(): + return cache_path.read_text() + + for attempt in range(MAX_ATTEMPTS): + self._throttle() + response = self._http.get(f"/{endpoint}", params=params) + self._last_request = self._monotonic() + + if response.status_code == 202: + if attempt < MAX_ATTEMPTS - 1: + self._sleep(QUEUE_BACKOFF[min(attempt, len(QUEUE_BACKOFF) - 1)]) + continue + if response.status_code in (429, 503): + if attempt < MAX_ATTEMPTS - 1: + backoff = 2.0 * (2**attempt) * (1 + self._rng.uniform(0, 0.5)) + self._sleep(backoff) + continue + + response.raise_for_status() + self.cache_dir.mkdir(parents=True, exist_ok=True) + cache_path.write_text(response.text) + return response.text + + raise BGGQueueTimeout( + f"BGG did not serve /{endpoint} after {MAX_ATTEMPTS} attempts " + "(still queued or throttled) — wait a minute and re-run; " + "completed work is cached." + ) + + # -- typed endpoint wrappers ------------------------------------------ + + def search( + self, query: str, types: str = "boardgame,boardgameexpansion" + ) -> list[SearchResult]: + return parse_search(self.get_xml("search", {"query": query, "type": types})) + + def things( + self, + ids: Iterable[int], + stats: bool = False, + versions: bool = False, + ) -> list[ThingDetails]: + params = {"id": ",".join(str(i) for i in ids)} + if stats: + params["stats"] = "1" + if versions: + params["versions"] = "1" + return parse_things(self.get_xml("thing", params)) + + def collection( + self, + username: str, + subtype: str | None = None, + version: bool = True, + ) -> list[CollectionItem]: + params = {"username": username, "own": "1"} + if subtype: + params["subtype"] = subtype + if version: + params["version"] = "1" + return parse_collection(self.get_xml("collection", params)) + + def collection_full(self, username: str) -> list[CollectionItem]: + """Owned items incl. expansions (excluded from the default subtype).""" + base = self.collection(username) + expansions = self.collection(username, subtype="boardgameexpansion") + seen = {item.coll_id for item in base} + return base + [e for e in expansions if e.coll_id not in seen] diff --git a/src/bggpipe/models.py b/src/bggpipe/models.py new file mode 100644 index 0000000..911be4b --- /dev/null +++ b/src/bggpipe/models.py @@ -0,0 +1,154 @@ +"""Dataclasses + XML parsers for the BGG XML API2 responses we consume.""" + +from __future__ import annotations + +import xml.etree.ElementTree as ET # element types only; parsing goes via defusedxml +from dataclasses import dataclass, field + +from defusedxml.ElementTree import fromstring as _safe_fromstring + + +class BGGResponseError(Exception): + """The API returned a well-formed error document (e.g. bad username).""" + + +@dataclass(frozen=True) +class SearchResult: + bgg_id: int + name: str + name_type: str # "primary" | "alternate" + year: int | None + type: str # "boardgame" | "boardgameexpansion" + + +@dataclass(frozen=True) +class GameVersion: + version_id: int + name: str + year: int | None + publishers: tuple[str, ...] + languages: tuple[str, ...] + + +@dataclass(frozen=True) +class ThingDetails: + bgg_id: int + name: str + year: int | None + type: str + owned: int | None = None + rank: int | None = None + versions: tuple[GameVersion, ...] = field(default=()) + + +@dataclass(frozen=True) +class CollectionItem: + object_id: int + coll_id: int + name: str + subtype: str + own: bool + year: int | None + version_id: int | None + + +def _root(xml_text: str) -> ET.Element: + root = _safe_fromstring(xml_text) + if root.tag == "errors": + message = root.findtext("./error/message") or "unknown BGG error" + raise BGGResponseError(message.strip()) + return root + + +def _attr_int(elem: ET.Element | None, attr: str = "value") -> int | None: + if elem is None: + return None + raw = elem.get(attr) + if raw is None or not raw.lstrip("-").isdigit(): + return None + return int(raw) + + +def parse_search(xml_text: str) -> list[SearchResult]: + results = [] + for item in _root(xml_text).findall("item"): + name_elem = item.find("name") + if name_elem is None or item.get("id") is None: + continue + results.append( + SearchResult( + bgg_id=int(item.get("id")), + name=name_elem.get("value", ""), + name_type=name_elem.get("type", "primary"), + year=_attr_int(item.find("yearpublished")), + type=item.get("type", "boardgame"), + ) + ) + return results + + +def _parse_version(item: ET.Element) -> GameVersion | None: + version_id = item.get("id") + name = item.find("name[@type='primary']") + if version_id is None: + return None + return GameVersion( + version_id=int(version_id), + name=name.get("value", "") if name is not None else "", + year=_attr_int(item.find("yearpublished")), + publishers=tuple( + link.get("value", "") + for link in item.findall("link[@type='boardgamepublisher']") + ), + languages=tuple( + link.get("value", "") for link in item.findall("link[@type='language']") + ), + ) + + +def parse_things(xml_text: str) -> list[ThingDetails]: + things = [] + for item in _root(xml_text).findall("item"): + name = item.find("name[@type='primary']") + rank_elem = item.find(".//ranks/rank[@name='boardgame']") + versions = [ + v + for v_item in item.findall("versions/item") + if (v := _parse_version(v_item)) is not None + ] + things.append( + ThingDetails( + bgg_id=int(item.get("id", 0)), + name=name.get("value", "") if name is not None else "", + year=_attr_int(item.find("yearpublished")), + type=item.get("type", "boardgame"), + owned=_attr_int(item.find(".//ratings/owned")), + rank=_attr_int(rank_elem), + versions=tuple(versions), + ) + ) + return things + + +def parse_collection(xml_text: str) -> list[CollectionItem]: + items = [] + for item in _root(xml_text).findall("item"): + status = item.find("status") + year_text = item.findtext("yearpublished") + version_item = item.find("version/item") + items.append( + CollectionItem( + object_id=int(item.get("objectid", 0)), + coll_id=int(item.get("collid", 0)), + name=item.findtext("name", default=""), + subtype=item.get("subtype", "boardgame"), + own=status is not None and status.get("own") == "1", + year=int(year_text) if year_text and year_text.isdigit() else None, + version_id=( + int(version_item.get("id")) + if version_item is not None and version_item.get("id") + else None + ), + ) + ) + return items diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..1e9d1a5 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import random + +import httpx +import pytest + +from bggpipe.bgg_client import BGGClient, BGGQueueTimeout, cache_key + +SEARCH_XML = ( + '' + '' + "" +) + + +class FakeClock: + def __init__(self) -> None: + self.now = 1000.0 + self.sleeps: list[float] = [] + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append(seconds) + self.now += seconds + + +def make_client(tmp_path, responses, clock=None): + """Client whose transport pops canned (status, body) responses.""" + calls = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request) + status, body = responses.pop(0) + return httpx.Response(status, text=body) + + clock = clock or FakeClock() + client = BGGClient( + cache_dir=tmp_path / "cache", + transport=httpx.MockTransport(handler), + sleep=clock.sleep, + monotonic=clock.monotonic, + rng=random.Random(42), + ) + return client, calls, clock + + +def test_caches_successful_responses(tmp_path): + client, calls, _ = make_client(tmp_path, [(200, SEARCH_XML)]) + first = client.get_xml("search", {"query": "catan", "type": "boardgame"}) + second = client.get_xml("search", {"query": "catan", "type": "boardgame"}) + assert first == second == SEARCH_XML + assert len(calls) == 1 # second call served from disk + cached = ( + tmp_path + / "cache" + / cache_key("search", {"query": "catan", "type": "boardgame"}) + ) + assert cached.exists() + + +def test_202_retries_with_spec_backoff_then_succeeds(tmp_path): + client, calls, clock = make_client( + tmp_path, [(202, ""), (202, ""), (200, SEARCH_XML)] + ) + xml = client.get_xml("collection", {"username": "someone", "own": "1"}) + assert xml == SEARCH_XML + assert len(calls) == 3 + assert clock.sleeps == [2.0, 5.0] # spec schedule between attempts + + +def test_202_gives_up_after_five_attempts(tmp_path): + client, calls, clock = make_client(tmp_path, [(202, "")] * 5) + with pytest.raises(BGGQueueTimeout): + client.get_xml("collection", {"username": "someone", "own": "1"}) + assert len(calls) == 5 + assert clock.sleeps == [2.0, 5.0, 10.0, 30.0] + # nothing cached on failure + assert not (tmp_path / "cache").exists() + + +def test_429_backs_off_with_jitter(tmp_path): + client, calls, clock = make_client(tmp_path, [(429, ""), (200, SEARCH_XML)]) + client.get_xml("search", {"query": "catan"}) + assert len(calls) == 2 + (backoff,) = clock.sleeps + assert 2.0 <= backoff <= 3.0 # base 2s, up to +50% jitter + + +def test_rate_limit_spaces_consecutive_requests(tmp_path): + client, calls, clock = make_client(tmp_path, [(200, SEARCH_XML), (200, SEARCH_XML)]) + client.get_xml("search", {"query": "catan"}) + client.get_xml("search", {"query": "wingspan"}) + # second request must wait the full 2s window (fake clock: no time passed) + assert clock.sleeps == [2.0] + + +def test_http_error_raises(tmp_path): + client, _, _ = make_client(tmp_path, [(500, "boom")]) + with pytest.raises(httpx.HTTPStatusError): + client.get_xml("search", {"query": "catan"}) + + +def test_cache_key_stable_and_filename_safe(tmp_path): + key1 = cache_key("search", {"query": "Café & Krieg?", "type": "boardgame"}) + key2 = cache_key("search", {"type": "boardgame", "query": "Café & Krieg?"}) + assert key1 == key2 # param order must not matter + assert "/" not in key1 and "?" not in key1 and key1.endswith(".xml") diff --git a/tests/test_models.py b/tests/test_models.py new file mode 100644 index 0000000..d32cefa --- /dev/null +++ b/tests/test_models.py @@ -0,0 +1,114 @@ +import pytest + +from bggpipe.models import ( + BGGResponseError, + parse_collection, + parse_search, + parse_things, +) + +SEARCH_XML = """ + + + + + + +""" + +THING_XML = """ + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + +COLLECTION_XML = """ + + Catan + 1995 + + + + Wingspan + + + +""" + +NOT_RANKED_XML = """ + + + + + + + +""" + +ERROR_XML = ( + "Invalid username specified" +) + + +def test_parse_search(): + results = parse_search(SEARCH_XML) + assert [r.bgg_id for r in results] == [13, 290448] + assert results[0].name == "CATAN" + assert results[0].year == 1995 + assert results[1].type == "boardgameexpansion" + assert results[1].name_type == "alternate" + assert results[1].year is None + + +def test_parse_things_with_stats_and_versions(): + (thing,) = parse_things(THING_XML) + assert thing.name == "Wingspan" # primary, not alternate + assert thing.owned == 123456 + assert thing.rank == 30 # boardgame rank, not the family rank + assert len(thing.versions) == 2 + english = thing.versions[0] + assert english.version_id == 465063 + assert english.publishers == ("Stonemaier Games",) + assert english.languages == ("English",) + + +def test_parse_things_not_ranked_is_none(): + (thing,) = parse_things(NOT_RANKED_XML) + assert thing.rank is None + assert thing.owned == 12 + + +def test_parse_collection(): + catan, wingspan = parse_collection(COLLECTION_XML) + assert (catan.object_id, catan.coll_id, catan.own) == (13, 101, True) + assert catan.version_id is None + assert wingspan.own is False # wishlist item must not count as owned + assert wingspan.version_id == 465063 + + +def test_error_document_raises(): + with pytest.raises(BGGResponseError, match="Invalid username"): + parse_collection(ERROR_XML) diff --git a/uv.lock b/uv.lock index 61be215..f9328af 100644 --- a/uv.lock +++ b/uv.lock @@ -29,6 +29,7 @@ name = "bggpipe" version = "0.1.0" source = { editable = "." } dependencies = [ + { name = "defusedxml" }, { name = "httpx" }, { name = "rapidfuzz" }, { name = "typer" }, @@ -42,6 +43,7 @@ dev = [ [package.metadata] requires-dist = [ + { name = "defusedxml", specifier = ">=0.7.1" }, { name = "httpx", specifier = ">=0.27" }, { name = "rapidfuzz", specifier = ">=3.9" }, { name = "typer", specifier = ">=0.12" }, @@ -71,6 +73,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "defusedxml" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, +] + [[package]] name = "h11" version = "0.16.0"