diff --git a/pyproject.toml b/pyproject.toml
index 2e84607..4832197 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,6 +7,7 @@ dependencies = [
"typer>=0.12",
"httpx>=0.27",
"rapidfuzz>=3.9",
+ "defusedxml>=0.7.1",
]
[project.scripts]
diff --git a/src/bggpipe/bgg_client.py b/src/bggpipe/bgg_client.py
new file mode 100644
index 0000000..40a5f38
--- /dev/null
+++ b/src/bggpipe/bgg_client.py
@@ -0,0 +1,145 @@
+"""BGG XML API2 client: disk cache, rate limiting, 202-queue and 429/503 retry.
+
+Every successful response is cached under data/bgg_cache/ keyed by
+endpoint + params, so re-runs never re-hit the API. The rate limiter
+guarantees ≤1 request every rate_limit_seconds to any BGG endpoint.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import random
+import re
+import time
+from collections.abc import Callable, Iterable
+from pathlib import Path
+from urllib.parse import urlencode
+
+import httpx
+
+from bggpipe.models import (
+ CollectionItem,
+ SearchResult,
+ ThingDetails,
+ parse_collection,
+ parse_search,
+ parse_things,
+)
+
+BASE_URL = "https://boardgamegeek.com/xmlapi2"
+QUEUE_BACKOFF = (2.0, 5.0, 10.0, 30.0) # sleeps between the 5 attempts (spec)
+MAX_ATTEMPTS = 5
+_UNSAFE = re.compile(r"[^A-Za-z0-9._=,-]+")
+
+
+class BGGQueueTimeout(Exception):
+ """BGG kept answering 202 (or throttling) past the retry budget."""
+
+
+def cache_key(endpoint: str, params: dict[str, str]) -> str:
+ query = urlencode(sorted(params.items()))
+ digest = hashlib.md5(f"{endpoint}?{query}".encode()).hexdigest()[:10]
+ slug = _UNSAFE.sub("-", query)[:80].strip("-")
+ return f"{endpoint}_{slug}_{digest}.xml"
+
+
+class BGGClient:
+ def __init__(
+ self,
+ cache_dir: Path,
+ rate_limit_seconds: float = 2.0,
+ transport: httpx.BaseTransport | None = None,
+ sleep: Callable[[float], None] = time.sleep,
+ monotonic: Callable[[], float] = time.monotonic,
+ rng: random.Random | None = None,
+ ) -> None:
+ self.cache_dir = cache_dir
+ self._rate = rate_limit_seconds
+ self._sleep = sleep
+ self._monotonic = monotonic
+ self._rng = rng or random.Random()
+ self._last_request: float | None = None
+ self._http = httpx.Client(
+ base_url=BASE_URL,
+ timeout=30.0,
+ headers={"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"},
+ transport=transport,
+ )
+
+ def _throttle(self) -> None:
+ if self._last_request is not None:
+ wait = self._rate - (self._monotonic() - self._last_request)
+ if wait > 0:
+ self._sleep(wait)
+
+ def get_xml(self, endpoint: str, params: dict[str, str]) -> str:
+ """Fetch one endpoint, serving from and filling the disk cache."""
+ cache_path = self.cache_dir / cache_key(endpoint, params)
+ if cache_path.exists():
+ return cache_path.read_text()
+
+ for attempt in range(MAX_ATTEMPTS):
+ self._throttle()
+ response = self._http.get(f"/{endpoint}", params=params)
+ self._last_request = self._monotonic()
+
+ if response.status_code == 202:
+ if attempt < MAX_ATTEMPTS - 1:
+ self._sleep(QUEUE_BACKOFF[min(attempt, len(QUEUE_BACKOFF) - 1)])
+ continue
+ if response.status_code in (429, 503):
+ if attempt < MAX_ATTEMPTS - 1:
+ backoff = 2.0 * (2**attempt) * (1 + self._rng.uniform(0, 0.5))
+ self._sleep(backoff)
+ continue
+
+ response.raise_for_status()
+ self.cache_dir.mkdir(parents=True, exist_ok=True)
+ cache_path.write_text(response.text)
+ return response.text
+
+ raise BGGQueueTimeout(
+ f"BGG did not serve /{endpoint} after {MAX_ATTEMPTS} attempts "
+ "(still queued or throttled) — wait a minute and re-run; "
+ "completed work is cached."
+ )
+
+ # -- typed endpoint wrappers ------------------------------------------
+
+ def search(
+ self, query: str, types: str = "boardgame,boardgameexpansion"
+ ) -> list[SearchResult]:
+ return parse_search(self.get_xml("search", {"query": query, "type": types}))
+
+ def things(
+ self,
+ ids: Iterable[int],
+ stats: bool = False,
+ versions: bool = False,
+ ) -> list[ThingDetails]:
+ params = {"id": ",".join(str(i) for i in ids)}
+ if stats:
+ params["stats"] = "1"
+ if versions:
+ params["versions"] = "1"
+ return parse_things(self.get_xml("thing", params))
+
+ def collection(
+ self,
+ username: str,
+ subtype: str | None = None,
+ version: bool = True,
+ ) -> list[CollectionItem]:
+ params = {"username": username, "own": "1"}
+ if subtype:
+ params["subtype"] = subtype
+ if version:
+ params["version"] = "1"
+ return parse_collection(self.get_xml("collection", params))
+
+ def collection_full(self, username: str) -> list[CollectionItem]:
+ """Owned items incl. expansions (excluded from the default subtype)."""
+ base = self.collection(username)
+ expansions = self.collection(username, subtype="boardgameexpansion")
+ seen = {item.coll_id for item in base}
+ return base + [e for e in expansions if e.coll_id not in seen]
diff --git a/src/bggpipe/models.py b/src/bggpipe/models.py
new file mode 100644
index 0000000..911be4b
--- /dev/null
+++ b/src/bggpipe/models.py
@@ -0,0 +1,154 @@
+"""Dataclasses + XML parsers for the BGG XML API2 responses we consume."""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET # element types only; parsing goes via defusedxml
+from dataclasses import dataclass, field
+
+from defusedxml.ElementTree import fromstring as _safe_fromstring
+
+
+class BGGResponseError(Exception):
+ """The API returned a well-formed error document (e.g. bad username)."""
+
+
+@dataclass(frozen=True)
+class SearchResult:
+ bgg_id: int
+ name: str
+ name_type: str # "primary" | "alternate"
+ year: int | None
+ type: str # "boardgame" | "boardgameexpansion"
+
+
+@dataclass(frozen=True)
+class GameVersion:
+ version_id: int
+ name: str
+ year: int | None
+ publishers: tuple[str, ...]
+ languages: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class ThingDetails:
+ bgg_id: int
+ name: str
+ year: int | None
+ type: str
+ owned: int | None = None
+ rank: int | None = None
+ versions: tuple[GameVersion, ...] = field(default=())
+
+
+@dataclass(frozen=True)
+class CollectionItem:
+ object_id: int
+ coll_id: int
+ name: str
+ subtype: str
+ own: bool
+ year: int | None
+ version_id: int | None
+
+
+def _root(xml_text: str) -> ET.Element:
+ root = _safe_fromstring(xml_text)
+ if root.tag == "errors":
+ message = root.findtext("./error/message") or "unknown BGG error"
+ raise BGGResponseError(message.strip())
+ return root
+
+
+def _attr_int(elem: ET.Element | None, attr: str = "value") -> int | None:
+ if elem is None:
+ return None
+ raw = elem.get(attr)
+ if raw is None or not raw.lstrip("-").isdigit():
+ return None
+ return int(raw)
+
+
+def parse_search(xml_text: str) -> list[SearchResult]:
+ results = []
+ for item in _root(xml_text).findall("item"):
+ name_elem = item.find("name")
+ if name_elem is None or item.get("id") is None:
+ continue
+ results.append(
+ SearchResult(
+ bgg_id=int(item.get("id")),
+ name=name_elem.get("value", ""),
+ name_type=name_elem.get("type", "primary"),
+ year=_attr_int(item.find("yearpublished")),
+ type=item.get("type", "boardgame"),
+ )
+ )
+ return results
+
+
+def _parse_version(item: ET.Element) -> GameVersion | None:
+ version_id = item.get("id")
+ name = item.find("name[@type='primary']")
+ if version_id is None:
+ return None
+ return GameVersion(
+ version_id=int(version_id),
+ name=name.get("value", "") if name is not None else "",
+ year=_attr_int(item.find("yearpublished")),
+ publishers=tuple(
+ link.get("value", "")
+ for link in item.findall("link[@type='boardgamepublisher']")
+ ),
+ languages=tuple(
+ link.get("value", "") for link in item.findall("link[@type='language']")
+ ),
+ )
+
+
+def parse_things(xml_text: str) -> list[ThingDetails]:
+ things = []
+ for item in _root(xml_text).findall("item"):
+ name = item.find("name[@type='primary']")
+ rank_elem = item.find(".//ranks/rank[@name='boardgame']")
+ versions = [
+ v
+ for v_item in item.findall("versions/item")
+ if (v := _parse_version(v_item)) is not None
+ ]
+ things.append(
+ ThingDetails(
+ bgg_id=int(item.get("id", 0)),
+ name=name.get("value", "") if name is not None else "",
+ year=_attr_int(item.find("yearpublished")),
+ type=item.get("type", "boardgame"),
+ owned=_attr_int(item.find(".//ratings/owned")),
+ rank=_attr_int(rank_elem),
+ versions=tuple(versions),
+ )
+ )
+ return things
+
+
+def parse_collection(xml_text: str) -> list[CollectionItem]:
+ items = []
+ for item in _root(xml_text).findall("item"):
+ status = item.find("status")
+ year_text = item.findtext("yearpublished")
+ version_item = item.find("version/item")
+ items.append(
+ CollectionItem(
+ object_id=int(item.get("objectid", 0)),
+ coll_id=int(item.get("collid", 0)),
+ name=item.findtext("name", default=""),
+ subtype=item.get("subtype", "boardgame"),
+ own=status is not None and status.get("own") == "1",
+ year=int(year_text) if year_text and year_text.isdigit() else None,
+ version_id=(
+ int(version_item.get("id"))
+ if version_item is not None and version_item.get("id")
+ else None
+ ),
+ )
+ )
+ return items
diff --git a/tests/test_client.py b/tests/test_client.py
new file mode 100644
index 0000000..1e9d1a5
--- /dev/null
+++ b/tests/test_client.py
@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import random
+
+import httpx
+import pytest
+
+from bggpipe.bgg_client import BGGClient, BGGQueueTimeout, cache_key
+
+SEARCH_XML = (
+ '- '
+ ''
+ "
"
+)
+
+
+class FakeClock:
+ def __init__(self) -> None:
+ self.now = 1000.0
+ self.sleeps: list[float] = []
+
+ def monotonic(self) -> float:
+ return self.now
+
+ def sleep(self, seconds: float) -> None:
+ self.sleeps.append(seconds)
+ self.now += seconds
+
+
+def make_client(tmp_path, responses, clock=None):
+ """Client whose transport pops canned (status, body) responses."""
+ calls = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ calls.append(request)
+ status, body = responses.pop(0)
+ return httpx.Response(status, text=body)
+
+ clock = clock or FakeClock()
+ client = BGGClient(
+ cache_dir=tmp_path / "cache",
+ transport=httpx.MockTransport(handler),
+ sleep=clock.sleep,
+ monotonic=clock.monotonic,
+ rng=random.Random(42),
+ )
+ return client, calls, clock
+
+
+def test_caches_successful_responses(tmp_path):
+ client, calls, _ = make_client(tmp_path, [(200, SEARCH_XML)])
+ first = client.get_xml("search", {"query": "catan", "type": "boardgame"})
+ second = client.get_xml("search", {"query": "catan", "type": "boardgame"})
+ assert first == second == SEARCH_XML
+ assert len(calls) == 1 # second call served from disk
+ cached = (
+ tmp_path
+ / "cache"
+ / cache_key("search", {"query": "catan", "type": "boardgame"})
+ )
+ assert cached.exists()
+
+
+def test_202_retries_with_spec_backoff_then_succeeds(tmp_path):
+ client, calls, clock = make_client(
+ tmp_path, [(202, ""), (202, ""), (200, SEARCH_XML)]
+ )
+ xml = client.get_xml("collection", {"username": "someone", "own": "1"})
+ assert xml == SEARCH_XML
+ assert len(calls) == 3
+ assert clock.sleeps == [2.0, 5.0] # spec schedule between attempts
+
+
+def test_202_gives_up_after_five_attempts(tmp_path):
+ client, calls, clock = make_client(tmp_path, [(202, "")] * 5)
+ with pytest.raises(BGGQueueTimeout):
+ client.get_xml("collection", {"username": "someone", "own": "1"})
+ assert len(calls) == 5
+ assert clock.sleeps == [2.0, 5.0, 10.0, 30.0]
+ # nothing cached on failure
+ assert not (tmp_path / "cache").exists()
+
+
+def test_429_backs_off_with_jitter(tmp_path):
+ client, calls, clock = make_client(tmp_path, [(429, ""), (200, SEARCH_XML)])
+ client.get_xml("search", {"query": "catan"})
+ assert len(calls) == 2
+ (backoff,) = clock.sleeps
+ assert 2.0 <= backoff <= 3.0 # base 2s, up to +50% jitter
+
+
+def test_rate_limit_spaces_consecutive_requests(tmp_path):
+ client, calls, clock = make_client(tmp_path, [(200, SEARCH_XML), (200, SEARCH_XML)])
+ client.get_xml("search", {"query": "catan"})
+ client.get_xml("search", {"query": "wingspan"})
+ # second request must wait the full 2s window (fake clock: no time passed)
+ assert clock.sleeps == [2.0]
+
+
+def test_http_error_raises(tmp_path):
+ client, _, _ = make_client(tmp_path, [(500, "boom")])
+ with pytest.raises(httpx.HTTPStatusError):
+ client.get_xml("search", {"query": "catan"})
+
+
+def test_cache_key_stable_and_filename_safe(tmp_path):
+ key1 = cache_key("search", {"query": "Café & Krieg?", "type": "boardgame"})
+ key2 = cache_key("search", {"type": "boardgame", "query": "Café & Krieg?"})
+ assert key1 == key2 # param order must not matter
+ assert "/" not in key1 and "?" not in key1 and key1.endswith(".xml")
diff --git a/tests/test_models.py b/tests/test_models.py
new file mode 100644
index 0000000..d32cefa
--- /dev/null
+++ b/tests/test_models.py
@@ -0,0 +1,114 @@
+import pytest
+
+from bggpipe.models import (
+ BGGResponseError,
+ parse_collection,
+ parse_search,
+ parse_things,
+)
+
+SEARCH_XML = """
+ -
+
+
+ -
+
+
+"""
+
+THING_XML = """
+ -
+
+
+
+
+
+
+
+
+
+
+
+
-
+
+
+
+
+
+ -
+
+
+
+
+
+
+
+"""
+
+COLLECTION_XML = """
+ -
+ Catan
+ 1995
+
+
+ -
+ Wingspan
+
+
+
+"""
+
+NOT_RANKED_XML = """
+ -
+
+
+
+
+
+
+"""
+
+ERROR_XML = (
+ "Invalid username specified"
+)
+
+
+def test_parse_search():
+ results = parse_search(SEARCH_XML)
+ assert [r.bgg_id for r in results] == [13, 290448]
+ assert results[0].name == "CATAN"
+ assert results[0].year == 1995
+ assert results[1].type == "boardgameexpansion"
+ assert results[1].name_type == "alternate"
+ assert results[1].year is None
+
+
+def test_parse_things_with_stats_and_versions():
+ (thing,) = parse_things(THING_XML)
+ assert thing.name == "Wingspan" # primary, not alternate
+ assert thing.owned == 123456
+ assert thing.rank == 30 # boardgame rank, not the family rank
+ assert len(thing.versions) == 2
+ english = thing.versions[0]
+ assert english.version_id == 465063
+ assert english.publishers == ("Stonemaier Games",)
+ assert english.languages == ("English",)
+
+
+def test_parse_things_not_ranked_is_none():
+ (thing,) = parse_things(NOT_RANKED_XML)
+ assert thing.rank is None
+ assert thing.owned == 12
+
+
+def test_parse_collection():
+ catan, wingspan = parse_collection(COLLECTION_XML)
+ assert (catan.object_id, catan.coll_id, catan.own) == (13, 101, True)
+ assert catan.version_id is None
+ assert wingspan.own is False # wishlist item must not count as owned
+ assert wingspan.version_id == 465063
+
+
+def test_error_document_raises():
+ with pytest.raises(BGGResponseError, match="Invalid username"):
+ parse_collection(ERROR_XML)
diff --git a/uv.lock b/uv.lock
index 61be215..f9328af 100644
--- a/uv.lock
+++ b/uv.lock
@@ -29,6 +29,7 @@ name = "bggpipe"
version = "0.1.0"
source = { editable = "." }
dependencies = [
+ { name = "defusedxml" },
{ name = "httpx" },
{ name = "rapidfuzz" },
{ name = "typer" },
@@ -42,6 +43,7 @@ dev = [
[package.metadata]
requires-dist = [
+ { name = "defusedxml", specifier = ">=0.7.1" },
{ name = "httpx", specifier = ">=0.27" },
{ name = "rapidfuzz", specifier = ">=3.9" },
{ name = "typer", specifier = ">=0.12" },
@@ -71,6 +73,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
]
+[[package]]
+name = "defusedxml"
+version = "0.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/d5/c66da9b79e5bdb124974bfe172b4daf3c984ebd9c2a06e2b8a4dc7331c72/defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69", size = 75520, upload-time = "2021-03-08T10:59:26.269Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" },
+]
+
[[package]]
name = "h11"
version = "0.16.0"