BGG API client: disk cache, rate limiting, 202-queue retry, XML parsing

httpx client with injectable clock/sleep/rng for testability. Successful
responses cached under data/bgg_cache/ keyed by endpoint+params; 202
retries follow the spec schedule (2/5/10/30s, give up after 5); 429/503
get jittered exponential backoff; consecutive requests are spaced
rate_limit_seconds apart. Parsers (via defusedxml, per security hook)
cover search, thing (+stats/+versions), and collection, including the
Not Ranked and error-document cases.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

pyproject.toml

@@ -7,6 +7,7 @@ dependencies = [
     "typer>=0.12",
     "httpx>=0.27",
     "rapidfuzz>=3.9",
+    "defusedxml>=0.7.1",
 ]
 
 [project.scripts]

src/bggpipe/bgg_client.py

@@ -0,0 +1,145 @@
+"""BGG XML API2 client: disk cache, rate limiting, 202-queue and 429/503 retry.
+
+Every successful response is cached under data/bgg_cache/ keyed by
+endpoint + params, so re-runs never re-hit the API. The rate limiter
+guarantees ≤1 request every rate_limit_seconds to any BGG endpoint.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import random
+import re
+import time
+from collections.abc import Callable, Iterable
+from pathlib import Path
+from urllib.parse import urlencode
+
+import httpx
+
+from bggpipe.models import (
+    CollectionItem,
+    SearchResult,
+    ThingDetails,
+    parse_collection,
+    parse_search,
+    parse_things,
+)
+
+BASE_URL = "https://boardgamegeek.com/xmlapi2"
+QUEUE_BACKOFF = (2.0, 5.0, 10.0, 30.0)  # sleeps between the 5 attempts (spec)
+MAX_ATTEMPTS = 5
+_UNSAFE = re.compile(r"[^A-Za-z0-9._=,-]+")
+
+
+class BGGQueueTimeout(Exception):
+    """BGG kept answering 202 (or throttling) past the retry budget."""
+
+
+def cache_key(endpoint: str, params: dict[str, str]) -> str:
+    query = urlencode(sorted(params.items()))
+    digest = hashlib.md5(f"{endpoint}?{query}".encode()).hexdigest()[:10]
+    slug = _UNSAFE.sub("-", query)[:80].strip("-")
+    return f"{endpoint}_{slug}_{digest}.xml"
+
+
+class BGGClient:
+    def __init__(
+        self,
+        cache_dir: Path,
+        rate_limit_seconds: float = 2.0,
+        transport: httpx.BaseTransport | None = None,
+        sleep: Callable[[float], None] = time.sleep,
+        monotonic: Callable[[], float] = time.monotonic,
+        rng: random.Random | None = None,
+    ) -> None:
+        self.cache_dir = cache_dir
+        self._rate = rate_limit_seconds
+        self._sleep = sleep
+        self._monotonic = monotonic
+        self._rng = rng or random.Random()
+        self._last_request: float | None = None
+        self._http = httpx.Client(
+            base_url=BASE_URL,
+            timeout=30.0,
+            headers={"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"},
+            transport=transport,
+        )
+
+    def _throttle(self) -> None:
+        if self._last_request is not None:
+            wait = self._rate - (self._monotonic() - self._last_request)
+            if wait > 0:
+                self._sleep(wait)
+
+    def get_xml(self, endpoint: str, params: dict[str, str]) -> str:
+        """Fetch one endpoint, serving from and filling the disk cache."""
+        cache_path = self.cache_dir / cache_key(endpoint, params)
+        if cache_path.exists():
+            return cache_path.read_text()
+
+        for attempt in range(MAX_ATTEMPTS):
+            self._throttle()
+            response = self._http.get(f"/{endpoint}", params=params)
+            self._last_request = self._monotonic()
+
+            if response.status_code == 202:
+                if attempt < MAX_ATTEMPTS - 1:
+                    self._sleep(QUEUE_BACKOFF[min(attempt, len(QUEUE_BACKOFF) - 1)])
+                continue
+            if response.status_code in (429, 503):
+                if attempt < MAX_ATTEMPTS - 1:
+                    backoff = 2.0 * (2**attempt) * (1 + self._rng.uniform(0, 0.5))
+                    self._sleep(backoff)
+                continue
+
+            response.raise_for_status()
+            self.cache_dir.mkdir(parents=True, exist_ok=True)
+            cache_path.write_text(response.text)
+            return response.text
+
+        raise BGGQueueTimeout(
+            f"BGG did not serve /{endpoint} after {MAX_ATTEMPTS} attempts "
+            "(still queued or throttled) — wait a minute and re-run; "
+            "completed work is cached."
+        )
+
+    # -- typed endpoint wrappers ------------------------------------------
+
+    def search(
+        self, query: str, types: str = "boardgame,boardgameexpansion"
+    ) -> list[SearchResult]:
+        return parse_search(self.get_xml("search", {"query": query, "type": types}))
+
+    def things(
+        self,
+        ids: Iterable[int],
+        stats: bool = False,
+        versions: bool = False,
+    ) -> list[ThingDetails]:
+        params = {"id": ",".join(str(i) for i in ids)}
+        if stats:
+            params["stats"] = "1"
+        if versions:
+            params["versions"] = "1"
+        return parse_things(self.get_xml("thing", params))
+
+    def collection(
+        self,
+        username: str,
+        subtype: str | None = None,
+        version: bool = True,
+    ) -> list[CollectionItem]:
+        params = {"username": username, "own": "1"}
+        if subtype:
+            params["subtype"] = subtype
+        if version:
+            params["version"] = "1"
+        return parse_collection(self.get_xml("collection", params))
+
+    def collection_full(self, username: str) -> list[CollectionItem]:
+        """Owned items incl. expansions (excluded from the default subtype)."""
+        base = self.collection(username)
+        expansions = self.collection(username, subtype="boardgameexpansion")
+        seen = {item.coll_id for item in base}
+        return base + [e for e in expansions if e.coll_id not in seen]

src/bggpipe/models.py

@@ -0,0 +1,154 @@
+"""Dataclasses + XML parsers for the BGG XML API2 responses we consume."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET  # element types only; parsing goes via defusedxml
+from dataclasses import dataclass, field
+
+from defusedxml.ElementTree import fromstring as _safe_fromstring
+
+
+class BGGResponseError(Exception):
+    """The API returned a well-formed error document (e.g. bad username)."""
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    bgg_id: int
+    name: str
+    name_type: str  # "primary" | "alternate"
+    year: int | None
+    type: str  # "boardgame" | "boardgameexpansion"
+
+
+@dataclass(frozen=True)
+class GameVersion:
+    version_id: int
+    name: str
+    year: int | None
+    publishers: tuple[str, ...]
+    languages: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ThingDetails:
+    bgg_id: int
+    name: str
+    year: int | None
+    type: str
+    owned: int | None = None
+    rank: int | None = None
+    versions: tuple[GameVersion, ...] = field(default=())
+
+
+@dataclass(frozen=True)
+class CollectionItem:
+    object_id: int
+    coll_id: int
+    name: str
+    subtype: str
+    own: bool
+    year: int | None
+    version_id: int | None
+
+
+def _root(xml_text: str) -> ET.Element:
+    root = _safe_fromstring(xml_text)
+    if root.tag == "errors":
+        message = root.findtext("./error/message") or "unknown BGG error"
+        raise BGGResponseError(message.strip())
+    return root
+
+
+def _attr_int(elem: ET.Element | None, attr: str = "value") -> int | None:
+    if elem is None:
+        return None
+    raw = elem.get(attr)
+    if raw is None or not raw.lstrip("-").isdigit():
+        return None
+    return int(raw)
+
+
+def parse_search(xml_text: str) -> list[SearchResult]:
+    results = []
+    for item in _root(xml_text).findall("item"):
+        name_elem = item.find("name")
+        if name_elem is None or item.get("id") is None:
+            continue
+        results.append(
+            SearchResult(
+                bgg_id=int(item.get("id")),
+                name=name_elem.get("value", ""),
+                name_type=name_elem.get("type", "primary"),
+                year=_attr_int(item.find("yearpublished")),
+                type=item.get("type", "boardgame"),
+            )
+        )
+    return results
+
+
+def _parse_version(item: ET.Element) -> GameVersion | None:
+    version_id = item.get("id")
+    name = item.find("name[@type='primary']")
+    if version_id is None:
+        return None
+    return GameVersion(
+        version_id=int(version_id),
+        name=name.get("value", "") if name is not None else "",
+        year=_attr_int(item.find("yearpublished")),
+        publishers=tuple(
+            link.get("value", "")
+            for link in item.findall("link[@type='boardgamepublisher']")
+        ),
+        languages=tuple(
+            link.get("value", "") for link in item.findall("link[@type='language']")
+        ),
+    )
+
+
+def parse_things(xml_text: str) -> list[ThingDetails]:
+    things = []
+    for item in _root(xml_text).findall("item"):
+        name = item.find("name[@type='primary']")
+        rank_elem = item.find(".//ranks/rank[@name='boardgame']")
+        versions = [
+            v
+            for v_item in item.findall("versions/item")
+            if (v := _parse_version(v_item)) is not None
+        ]
+        things.append(
+            ThingDetails(
+                bgg_id=int(item.get("id", 0)),
+                name=name.get("value", "") if name is not None else "",
+                year=_attr_int(item.find("yearpublished")),
+                type=item.get("type", "boardgame"),
+                owned=_attr_int(item.find(".//ratings/owned")),
+                rank=_attr_int(rank_elem),
+                versions=tuple(versions),
+            )
+        )
+    return things
+
+
+def parse_collection(xml_text: str) -> list[CollectionItem]:
+    items = []
+    for item in _root(xml_text).findall("item"):
+        status = item.find("status")
+        year_text = item.findtext("yearpublished")
+        version_item = item.find("version/item")
+        items.append(
+            CollectionItem(
+                object_id=int(item.get("objectid", 0)),
+                coll_id=int(item.get("collid", 0)),
+                name=item.findtext("name", default=""),
+                subtype=item.get("subtype", "boardgame"),
+                own=status is not None and status.get("own") == "1",
+                year=int(year_text) if year_text and year_text.isdigit() else None,
+                version_id=(
+                    int(version_item.get("id"))
+                    if version_item is not None and version_item.get("id")
+                    else None
+                ),
+            )
+        )
+    return items

tests/test_client.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import random
+
+import httpx
+import pytest
+
+from bggpipe.bgg_client import BGGClient, BGGQueueTimeout, cache_key
+
+SEARCH_XML = (
+    '<items total="1"><item type="boardgame" id="13">'
+    '<name type="primary" value="CATAN"/><yearpublished value="1995"/>'
+    "</item></items>"
+)
+
+
+class FakeClock:
+    def __init__(self) -> None:
+        self.now = 1000.0
+        self.sleeps: list[float] = []
+
+    def monotonic(self) -> float:
+        return self.now
+
+    def sleep(self, seconds: float) -> None:
+        self.sleeps.append(seconds)
+        self.now += seconds
+
+
+def make_client(tmp_path, responses, clock=None):
+    """Client whose transport pops canned (status, body) responses."""
+    calls = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(request)
+        status, body = responses.pop(0)
+        return httpx.Response(status, text=body)
+
+    clock = clock or FakeClock()
+    client = BGGClient(
+        cache_dir=tmp_path / "cache",
+        transport=httpx.MockTransport(handler),
+        sleep=clock.sleep,
+        monotonic=clock.monotonic,
+        rng=random.Random(42),
+    )
+    return client, calls, clock
+
+
+def test_caches_successful_responses(tmp_path):
+    client, calls, _ = make_client(tmp_path, [(200, SEARCH_XML)])
+    first = client.get_xml("search", {"query": "catan", "type": "boardgame"})
+    second = client.get_xml("search", {"query": "catan", "type": "boardgame"})
+    assert first == second == SEARCH_XML
+    assert len(calls) == 1  # second call served from disk
+    cached = (
+        tmp_path
+        / "cache"
+        / cache_key("search", {"query": "catan", "type": "boardgame"})
+    )
+    assert cached.exists()
+
+
+def test_202_retries_with_spec_backoff_then_succeeds(tmp_path):
+    client, calls, clock = make_client(
+        tmp_path, [(202, ""), (202, ""), (200, SEARCH_XML)]
+    )
+    xml = client.get_xml("collection", {"username": "someone", "own": "1"})
+    assert xml == SEARCH_XML
+    assert len(calls) == 3
+    assert clock.sleeps == [2.0, 5.0]  # spec schedule between attempts
+
+
+def test_202_gives_up_after_five_attempts(tmp_path):
+    client, calls, clock = make_client(tmp_path, [(202, "")] * 5)
+    with pytest.raises(BGGQueueTimeout):
+        client.get_xml("collection", {"username": "someone", "own": "1"})
+    assert len(calls) == 5
+    assert clock.sleeps == [2.0, 5.0, 10.0, 30.0]
+    # nothing cached on failure
+    assert not (tmp_path / "cache").exists()
+
+
+def test_429_backs_off_with_jitter(tmp_path):
+    client, calls, clock = make_client(tmp_path, [(429, ""), (200, SEARCH_XML)])
+    client.get_xml("search", {"query": "catan"})
+    assert len(calls) == 2
+    (backoff,) = clock.sleeps
+    assert 2.0 <= backoff <= 3.0  # base 2s, up to +50% jitter
+
+
+def test_rate_limit_spaces_consecutive_requests(tmp_path):
+    client, calls, clock = make_client(tmp_path, [(200, SEARCH_XML), (200, SEARCH_XML)])
+    client.get_xml("search", {"query": "catan"})
+    client.get_xml("search", {"query": "wingspan"})
+    # second request must wait the full 2s window (fake clock: no time passed)
+    assert clock.sleeps == [2.0]
+
+
+def test_http_error_raises(tmp_path):
+    client, _, _ = make_client(tmp_path, [(500, "boom")])
+    with pytest.raises(httpx.HTTPStatusError):
+        client.get_xml("search", {"query": "catan"})
+
+
+def test_cache_key_stable_and_filename_safe(tmp_path):
+    key1 = cache_key("search", {"query": "Café & Krieg?", "type": "boardgame"})
+    key2 = cache_key("search", {"type": "boardgame", "query": "Café & Krieg?"})
+    assert key1 == key2  # param order must not matter
+    assert "/" not in key1 and "?" not in key1 and key1.endswith(".xml")

tests/test_models.py

@@ -0,0 +1,114 @@
+import pytest
+
+from bggpipe.models import (
+    BGGResponseError,
+    parse_collection,
+    parse_search,
+    parse_things,
+)
+
+SEARCH_XML = """<items total="2">
+  <item type="boardgame" id="13">
+    <name type="primary" value="CATAN"/><yearpublished value="1995"/>
+  </item>
+  <item type="boardgameexpansion" id="290448">
+    <name type="alternate" value="Wingspan: Europa"/>
+  </item>
+</items>"""
+
+THING_XML = """<items>
+  <item type="boardgame" id="266192">
+    <name type="primary" sortindex="1" value="Wingspan"/>
+    <name type="alternate" sortindex="1" value="Flügelschlag"/>
+    <yearpublished value="2019"/>
+    <statistics page="1"><ratings>
+      <owned value="123456"/>
+      <ranks>
+        <rank type="subtype" id="1" name="boardgame" value="30"/>
+        <rank type="family" id="5497" name="strategygames" value="25"/>
+      </ranks>
+    </ratings></statistics>
+    <versions>
+      <item type="boardgameversion" id="465063">
+        <name type="primary" value="English edition"/>
+        <yearpublished value="2019"/>
+        <link type="boardgamepublisher" id="23202" value="Stonemaier Games"/>
+        <link type="language" id="2184" value="English"/>
+      </item>
+      <item type="boardgameversion" id="465064">
+        <name type="primary" value="German edition"/>
+        <yearpublished value="2019"/>
+        <link type="boardgamepublisher" id="22160" value="Feuerland Spiele"/>
+        <link type="language" id="2188" value="German"/>
+      </item>
+    </versions>
+  </item>
+</items>"""
+
+COLLECTION_XML = """<items totalitems="2">
+  <item objecttype="thing" objectid="13" subtype="boardgame" collid="101">
+    <name sortindex="1">Catan</name>
+    <yearpublished>1995</yearpublished>
+    <status own="1" wanttoplay="0"/>
+  </item>
+  <item objecttype="thing" objectid="266192" subtype="boardgame" collid="102">
+    <name sortindex="1">Wingspan</name>
+    <status own="0" wishlist="1"/>
+    <version><item type="boardgameversion" id="465063"/></version>
+  </item>
+</items>"""
+
+NOT_RANKED_XML = """<items>
+  <item type="boardgame" id="99999">
+    <name type="primary" value="Obscurity"/>
+    <statistics><ratings>
+      <owned value="12"/>
+      <ranks><rank type="subtype" id="1" name="boardgame" value="Not Ranked"/></ranks>
+    </ratings></statistics>
+  </item>
+</items>"""
+
+ERROR_XML = (
+    "<errors><error><message>Invalid username specified</message></error></errors>"
+)
+
+
+def test_parse_search():
+    results = parse_search(SEARCH_XML)
+    assert [r.bgg_id for r in results] == [13, 290448]
+    assert results[0].name == "CATAN"
+    assert results[0].year == 1995
+    assert results[1].type == "boardgameexpansion"
+    assert results[1].name_type == "alternate"
+    assert results[1].year is None
+
+
+def test_parse_things_with_stats_and_versions():
+    (thing,) = parse_things(THING_XML)
+    assert thing.name == "Wingspan"  # primary, not alternate
+    assert thing.owned == 123456
+    assert thing.rank == 30  # boardgame rank, not the family rank
+    assert len(thing.versions) == 2
+    english = thing.versions[0]
+    assert english.version_id == 465063
+    assert english.publishers == ("Stonemaier Games",)
+    assert english.languages == ("English",)
+
+
+def test_parse_things_not_ranked_is_none():
+    (thing,) = parse_things(NOT_RANKED_XML)
+    assert thing.rank is None
+    assert thing.owned == 12
+
+
+def test_parse_collection():
+    catan, wingspan = parse_collection(COLLECTION_XML)
+    assert (catan.object_id, catan.coll_id, catan.own) == (13, 101, True)
+    assert catan.version_id is None
+    assert wingspan.own is False  # wishlist item must not count as owned
+    assert wingspan.version_id == 465063
+
+
+def test_error_document_raises():
+    with pytest.raises(BGGResponseError, match="Invalid username"):
+        parse_collection(ERROR_XML)

uv.lock

@@ -29,6 +29,7 @@ name = "bggpipe"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [
+    { name = "defusedxml" },
     { name = "httpx" },
     { name = "rapidfuzz" },
     { name = "typer" },
@@ -42,6 +43,7 @@ dev = [
 
 [package.metadata]
 requires-dist = [
+    { name = "defusedxml", specifier = ">=0.7.1" },
     { name = "httpx", specifier = ">=0.27" },
     { name = "rapidfuzz", specifier = ">=3.9" },
     { name = "typer", specifier = ">=0.12" },
@@ -71,6 +73,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
 ]
 
+[[package]]
+name = "defusedxml"
+version = "0.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" },
+]
+
 [[package]]
 name = "h11"
 version = "0.16.0"