From 2109e3544aa8ff4a08c89df4a66a7f7edbe9c796 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Sat, 1 Aug 2026 12:33:32 -0400 Subject: [PATCH] Resolve stage: matching, version resolution, fixtures; BGG API auth bggpipe resolve works end to end: search -> exact/fuzzy candidate scoring -> auto/ambiguous/unmatched classification with owned-count tie-breaks (mixed base/expansion candidates never auto-match), version scoring from edition cues (never guessed; no cues -> version_unknown), idempotent matches.csv appends. Discovered mid-build: BGG now requires registered-application Bearer tokens on the XML API (2025 policy change) and returns 401 otherwise. Client sends Authorization from BGG_API_TOKEN and raises an actionable BGGAuthError; CLAUDE.md and the bgg-api skill are updated to match. Live fixture recording is blocked until registration is approved, so tests replay hand-crafted stub fixtures via a network-refusing transport; scripts/record_fixtures.py re-records real XML under the same cache keys once a token exists. One live read-only smoke test is skipped unless --run-live. Co-Authored-By: Claude Fable 5 --- .claude/skills/bgg-api/SKILL.md | 13 + CLAUDE.md | 3 +- data/matches.csv | 8 + data/titles.json | 40 ++ scripts/record_fixtures.py | 45 +++ scripts/write_stub_fixtures.py | 118 ++++++ src/bggpipe/bgg_client.py | 20 +- src/bggpipe/resolve.py | 350 +++++++++++++++++- ...rdgame-2Cboardgameexpansion_22856aa961.xml | 1 + ...rdgame-2Cboardgameexpansion_d875abc790.xml | 1 + ...rdgame-2Cboardgameexpansion_5be385814d.xml | 1 + ...rdgame-2Cboardgameexpansion_9f48c481c4.xml | 1 + ...rdgame-2Cboardgameexpansion_85d6b5d538.xml | 1 + ...rdgame-2Cboardgameexpansion_f5cb460ffe.xml | 1 + ...rdgame-2Cboardgameexpansion_0316afc400.xml | 1 + .../thing_id=266192-versions=1_4904563f7f.xml | 25 ++ ...ing_id=478-2C205398-stats=1_9bd49e518b.xml | 14 + tests/test_client.py | 23 ++ tests/test_resolve.py | 189 ++++++++++ tests/test_smoke_live.py | 31 ++ 20 files changed, 880 insertions(+), 6 deletions(-) create mode 100644 data/matches.csv create mode 100644 data/titles.json create mode 100644 scripts/record_fixtures.py create mode 100644 scripts/write_stub_fixtures.py create mode 100644 tests/fixtures/bgg_cache/search_query=Blorvath-3A-Quest-of-the-Zzyzx-type=boardgame-2Cboardgameexpansion_22856aa961.xml create mode 100644 tests/fixtures/bgg_cache/search_query=Caf-C3-A9-International-type=boardgame-2Cboardgameexpansion_d875abc790.xml create mode 100644 tests/fixtures/bgg_cache/search_query=Catan-type=boardgame-2Cboardgameexpansion_5be385814d.xml create mode 100644 tests/fixtures/bgg_cache/search_query=Citadels-type=boardgame-2Cboardgameexpansion_9f48c481c4.xml create mode 100644 tests/fixtures/bgg_cache/search_query=Wingspan-3A-European-Expansion-type=boardgame-2Cboardgameexpansion_85d6b5d538.xml create mode 100644 tests/fixtures/bgg_cache/search_query=Wingspan-Europe-type=boardgame-2Cboardgameexpansion_f5cb460ffe.xml create mode 100644 tests/fixtures/bgg_cache/search_query=Wingspan-type=boardgame-2Cboardgameexpansion_0316afc400.xml create mode 100644 tests/fixtures/bgg_cache/thing_id=266192-versions=1_4904563f7f.xml create mode 100644 tests/fixtures/bgg_cache/thing_id=478-2C205398-stats=1_9bd49e518b.xml create mode 100644 tests/test_resolve.py create mode 100644 tests/test_smoke_live.py diff --git a/.claude/skills/bgg-api/SKILL.md b/.claude/skills/bgg-api/SKILL.md index e00fa60..850dc31 100644 --- a/.claude/skills/bgg-api/SKILL.md +++ b/.claude/skills/bgg-api/SKILL.md @@ -5,6 +5,19 @@ description: Reference for BoardGameGeek's XML API2 and website automation — e # BoardGameGeek API & site automation reference +## Authentication (required since 2025) + +Every XML API request must carry `Authorization: Bearer ` or BGG +returns **401 Unauthorized**. Tokens come from a registered application: +create one at `https://boardgamegeek.com/applications` (non-commercial +license is free; approval can take a week or more), then generate a token +under "Tokens". `bggpipe` reads it from the `BGG_API_TOKEN` env var — never +put it in config.toml, code, or logs. Requests must go to +`boardgamegeek.com` **without** a leading `www` or the token is ignored. +Exception: downloading your own collection while logged in on the website +needs no registration — relevant to the Playwright stages, not the API +client. Usage is monitored per-application at `/applications` → "Usage". + ## Endpoints (XML API2 — the only sanctioned read API) - Search: `https://boardgamegeek.com/xmlapi2/search?query=&type=boardgame,boardgameexpansion` diff --git a/CLAUDE.md b/CLAUDE.md index 8ad4e55..03e1044 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,7 +15,8 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Hard rules (from spec — never violate) - **≤1 request every 2 seconds** to any BGG endpoint; jittered backoff on 429/503. Upload stage: 2–4 s randomized delay between games. -- **Credentials never touch disk or logs.** `ANTHROPIC_API_KEY`, `BGG_USERNAME`, `BGG_PASSWORD` come from env vars only. Playwright storage state is credential-adjacent — keep it gitignored. +- **Credentials never touch disk or logs.** `ANTHROPIC_API_KEY`, `BGG_USERNAME`, `BGG_PASSWORD`, `BGG_API_TOKEN` come from env vars only. Playwright storage state is credential-adjacent — keep it gitignored. +- **The XML API requires a registered app token** (`Authorization: Bearer`, from `BGG_API_TOKEN`) — unregistered requests get 401. Until Eric's registration at boardgamegeek.com/applications is approved, tests run on the stub fixtures in `tests/fixtures/bgg_cache/`; re-record them with `scripts/record_fixtures.py` once the token exists. - **Every stage is idempotent and resumable** — killing mid-run and restarting must lose no work; re-runs skip already-processed items. - Use only the XML API2 and the public website — no undocumented BGG endpoints (BGG tightened access policies in 2025). - BGG has **no write API**: writes drive the real website with a logged-in Playwright session. diff --git a/data/matches.csv b/data/matches.csv new file mode 100644 index 0000000..f9d4a09 --- /dev/null +++ b/data/matches.csv @@ -0,0 +1,8 @@ +title_raw,bgg_id,bgg_name,year,type,match_status,version_id,version_name,version_status,candidates_json,version_candidates_json,source_photos +Catan,13,CATAN,1995,boardgame,auto,,,version_unknown,"[{""bgg_id"": 13, ""name"": ""CATAN"", ""year"": 1995, ""type"": ""boardgame"", ""exact"": true, ""fuzzy"": 100.0, ""owned"": null, ""rank"": null}]",[],hand-typed-test-list +Wingspan,266192,Wingspan,2019,boardgame,auto,465063,English first edition,version_auto,"[{""bgg_id"": 266192, ""name"": ""Wingspan"", ""year"": 2019, ""type"": ""boardgame"", ""exact"": true, ""fuzzy"": 100.0, ""owned"": null, ""rank"": null}]","[{""version_id"": 465063, ""name"": ""English first edition"", ""year"": 2019, ""publishers"": [""Stonemaier Games""], ""languages"": [""English""], ""score"": 5}, {""version_id"": 521212, ""name"": ""English fourth printing"", ""year"": 2020, ""publishers"": [""Stonemaier Games""], ""languages"": [""English""], ""score"": 3}, {""version_id"": 472430, ""name"": ""German edition"", ""year"": 2019, ""publishers"": [""Feuerland Spiele""], ""languages"": [""German""], ""score"": 2}]",hand-typed-test-list +Wingspan: European Expansion,290448,Wingspan: European Expansion,2019,boardgameexpansion,auto,,,version_unknown,"[{""bgg_id"": 290448, ""name"": ""Wingspan: European Expansion"", ""year"": 2019, ""type"": ""boardgameexpansion"", ""exact"": true, ""fuzzy"": 100.0, ""owned"": null, ""rank"": null}]",[],hand-typed-test-list +Wingspan Europe,,,,,unmatched,,,,[],[],hand-typed-test-list +Café International,373,Café International,1989,boardgame,auto,,,version_unknown,"[{""bgg_id"": 373, ""name"": ""Café International"", ""year"": 1989, ""type"": ""boardgame"", ""exact"": true, ""fuzzy"": 100.0, ""owned"": null, ""rank"": null}]",[],hand-typed-test-list +Citadels,,,,,ambiguous,,,,"[{""bgg_id"": 478, ""name"": ""Citadels"", ""year"": 2000, ""type"": ""boardgame"", ""exact"": true, ""fuzzy"": 100.0, ""owned"": 85000, ""rank"": 250}, {""bgg_id"": 205398, ""name"": ""Citadels"", ""year"": 2016, ""type"": ""boardgame"", ""exact"": true, ""fuzzy"": 100.0, ""owned"": 24000, ""rank"": 400}]",[],hand-typed-test-list +Blorvath: Quest of the Zzyzx,,,,,unmatched,,,,[],[],hand-typed-test-list diff --git a/data/titles.json b/data/titles.json new file mode 100644 index 0000000..e25d4e6 --- /dev/null +++ b/data/titles.json @@ -0,0 +1,40 @@ +[ + { + "title_raw": "Catan", + "confidence": "high", + "source_photos": ["hand-typed-test-list"] + }, + { + "title_raw": "Wingspan", + "confidence": "high", + "publisher_hint": "Stonemaier Games", + "year_hint": 2019, + "language_hint": "English", + "source_photos": ["hand-typed-test-list"] + }, + { + "title_raw": "Wingspan: European Expansion", + "confidence": "high", + "source_photos": ["hand-typed-test-list"] + }, + { + "title_raw": "Wingspan Europe", + "confidence": "medium", + "source_photos": ["hand-typed-test-list"] + }, + { + "title_raw": "Café International", + "confidence": "high", + "source_photos": ["hand-typed-test-list"] + }, + { + "title_raw": "Citadels", + "confidence": "high", + "source_photos": ["hand-typed-test-list"] + }, + { + "title_raw": "Blorvath: Quest of the Zzyzx", + "confidence": "low", + "source_photos": ["hand-typed-test-list"] + } +] diff --git a/scripts/record_fixtures.py b/scripts/record_fixtures.py new file mode 100644 index 0000000..f1042d8 --- /dev/null +++ b/scripts/record_fixtures.py @@ -0,0 +1,45 @@ +"""Record the BGG XML fixtures the test suite replays. + +Runs the real resolve logic with the cache pointed at tests/fixtures/ +bgg_cache/, so exactly the responses resolve needs get recorded (live, +rate-limited). Run once; commit the fixtures. Re-running only fetches +whatever is not already recorded. + +Usage: uv run python scripts/record_fixtures.py +""" + +from __future__ import annotations + +import os +from pathlib import Path + +from bggpipe.bgg_client import BGGClient +from bggpipe.resolve import load_titles, resolve_entry + +FIXTURE_CACHE = Path("tests/fixtures/bgg_cache") + + +def main() -> None: + if not os.environ.get("BGG_API_TOKEN"): + raise SystemExit( + "BGG_API_TOKEN is not set. The XML API requires a registered " + "application token (https://boardgamegeek.com/applications). " + "Until one exists, the committed fixtures are the hand-crafted " + "stubs from scripts/write_stub_fixtures.py; delete " + f"{FIXTURE_CACHE} and re-run this script to replace them with " + "real recordings." + ) + client = BGGClient(cache_dir=FIXTURE_CACHE) + for entry in load_titles(Path("data/titles.json")): + row = resolve_entry(client, entry) + print( + f"{entry.title_raw!r}: {row.match_status} " + f"{row.bgg_name or '-'} ({row.bgg_id or '-'}) {row.version_status}" + ) + print(f"\nFixtures recorded in {FIXTURE_CACHE}:") + for f in sorted(FIXTURE_CACHE.iterdir()): + print(f" {f.name}") + + +if __name__ == "__main__": + main() diff --git a/scripts/write_stub_fixtures.py b/scripts/write_stub_fixtures.py new file mode 100644 index 0000000..3baabfd --- /dev/null +++ b/scripts/write_stub_fixtures.py @@ -0,0 +1,118 @@ +"""Write hand-crafted stub XML fixtures into tests/fixtures/bgg_cache/. + +BGG's XML API now requires a registered application token, which we don't +have yet, so these fixtures are realistic approximations of real responses +(shapes verified against the API docs) rather than recordings. Once a +BGG_API_TOKEN exists: delete tests/fixtures/bgg_cache/ and run +scripts/record_fixtures.py — it writes real recordings under the SAME +filenames, and the test suite must still pass. + +Usage: uv run python scripts/write_stub_fixtures.py +""" + +from __future__ import annotations + +from pathlib import Path + +from bggpipe.bgg_client import cache_key + +FIXTURE_CACHE = Path("tests/fixtures/bgg_cache") +SEARCH_TYPES = "boardgame,boardgameexpansion" + + +def search_item(bgg_id: int, name: str, year: int | None, type_: str) -> str: + year_xml = f'<yearpublished value="{year}"/>' if year else "" + return ( + f'<item type="{type_}" id="{bgg_id}">' + f'<name type="primary" value="{name}"/>{year_xml}</item>' + ) + + +WINGSPAN_FAMILY = ( + search_item(266192, "Wingspan", 2019, "boardgame") + + search_item(290448, "Wingspan: European Expansion", 2019, "boardgameexpansion") + + search_item(300580, "Wingspan: Oceania Expansion", 2020, "boardgameexpansion") + + search_item(366161, "Wingspan Asia", 2022, "boardgame") +) + +SEARCHES = { + "Catan": ( + search_item(13, "CATAN", 1995, "boardgame") + + search_item(926, "CATAN: Seafarers", 1997, "boardgameexpansion") + + search_item(27760, "Catan Dice Game", 2007, "boardgame") + + search_item(133038, "CATAN 3D Collector's Edition", 2005, "boardgame") + ), + "Wingspan": WINGSPAN_FAMILY, + "Wingspan: European Expansion": WINGSPAN_FAMILY, + "Wingspan Europe": WINGSPAN_FAMILY, + "Café International": ( + search_item(373, "Café International", 1989, "boardgame") + + search_item(8103, "Café International: Das Kartenspiel", 2001, "boardgame") + ), + "Citadels": ( + search_item(478, "Citadels", 2000, "boardgame") + + search_item(205398, "Citadels", 2016, "boardgame") + ), + "Blorvath: Quest of the Zzyzx": "", +} + +THINGS = { + # Citadels tie-break: neither edition dominates 10x -> stays ambiguous. + ("478,205398", "stats"): """<items> + <item type="boardgame" id="478"> + <name type="primary" value="Citadels"/><yearpublished value="2000"/> + <statistics><ratings><owned value="85000"/> + <ranks><rank type="subtype" id="1" name="boardgame" value="250"/></ranks> + </ratings></statistics> + </item> + <item type="boardgame" id="205398"> + <name type="primary" value="Citadels"/><yearpublished value="2016"/> + <statistics><ratings><owned value="24000"/> + <ranks><rank type="subtype" id="1" name="boardgame" value="400"/></ranks> + </ratings></statistics> + </item> +</items>""", + # Wingspan versions: 2019 Stonemaier English must beat the 2020 printing. + ("266192", "versions"): """<items> + <item type="boardgame" id="266192"> + <name type="primary" value="Wingspan"/><yearpublished value="2019"/> + <versions> + <item type="boardgameversion" id="465063"> + <name type="primary" value="English first edition"/> + <yearpublished value="2019"/> + <link type="boardgamepublisher" id="23202" value="Stonemaier Games"/> + <link type="language" id="2184" value="English"/> + </item> + <item type="boardgameversion" id="521212"> + <name type="primary" value="English fourth printing"/> + <yearpublished value="2020"/> + <link type="boardgamepublisher" id="23202" value="Stonemaier Games"/> + <link type="language" id="2184" value="English"/> + </item> + <item type="boardgameversion" id="472430"> + <name type="primary" value="German edition"/> + <yearpublished value="2019"/> + <link type="boardgamepublisher" id="22160" value="Feuerland Spiele"/> + <link type="language" id="2188" value="German"/> + </item> + </versions> + </item> +</items>""", +} + + +def main() -> None: + FIXTURE_CACHE.mkdir(parents=True, exist_ok=True) + for query, items in SEARCHES.items(): + key = cache_key("search", {"query": query, "type": SEARCH_TYPES}) + total = items.count("<item ") + (FIXTURE_CACHE / key).write_text(f'<items total="{total}">{items}</items>') + for (ids, flavor), xml in THINGS.items(): + key = cache_key("thing", {"id": ids, flavor: "1"}) + (FIXTURE_CACHE / key).write_text(xml) + for f in sorted(FIXTURE_CACHE.iterdir()): + print(f" {f.name}") + + +if __name__ == "__main__": + main() diff --git a/src/bggpipe/bgg_client.py b/src/bggpipe/bgg_client.py index 40a5f38..e148ffe 100644 --- a/src/bggpipe/bgg_client.py +++ b/src/bggpipe/bgg_client.py @@ -8,6 +8,7 @@ guarantees ≤1 request every rate_limit_seconds to any BGG endpoint. from __future__ import annotations import hashlib +import os import random import re import time @@ -36,6 +37,10 @@ class BGGQueueTimeout(Exception): """BGG kept answering 202 (or throttling) past the retry budget.""" +class BGGAuthError(Exception): + """BGG rejected the request as unauthorized (missing/invalid API token).""" + + def cache_key(endpoint: str, params: dict[str, str]) -> str: query = urlencode(sorted(params.items())) digest = hashlib.md5(f"{endpoint}?{query}".encode()).hexdigest()[:10] @@ -59,10 +64,16 @@ class BGGClient: self._monotonic = monotonic self._rng = rng or random.Random() self._last_request: float | None = None + headers = {"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"} + # BGG requires registered applications since 2025: the token from + # https://boardgamegeek.com/applications must accompany every request. + # Env var only — never config, disk, or logs. + if token := os.environ.get("BGG_API_TOKEN"): + headers["Authorization"] = f"Bearer {token}" self._http = httpx.Client( base_url=BASE_URL, timeout=30.0, - headers={"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"}, + headers=headers, transport=transport, ) @@ -87,6 +98,13 @@ class BGGClient: if attempt < MAX_ATTEMPTS - 1: self._sleep(QUEUE_BACKOFF[min(attempt, len(QUEUE_BACKOFF) - 1)]) continue + if response.status_code == 401: + raise BGGAuthError( + "BGG returned 401 Unauthorized. The XML API requires a " + "registered application token since 2025: register at " + "https://boardgamegeek.com/applications, create a token, " + "and export it as BGG_API_TOKEN." + ) if response.status_code in (429, 503): if attempt < MAX_ATTEMPTS - 1: backoff = 2.0 * (2**attempt) * (1 + self._rng.uniform(0, 0.5)) diff --git a/src/bggpipe/resolve.py b/src/bggpipe/resolve.py index fd541bf..90b1f3b 100644 --- a/src/bggpipe/resolve.py +++ b/src/bggpipe/resolve.py @@ -1,12 +1,354 @@ -"""Stage 2 — resolve extracted titles to BGG IDs and versions.""" +"""Stage 2 — resolve extracted titles to BGG IDs and versions. + +Reads data/titles.json, queries BGG search (+ thing stats for tie-breaks, ++ versions once a game is settled), classifies each title auto/ambiguous/ +unmatched, and appends rows to data/matches.csv. Re-runs skip titles +already present in matches.csv unless --force. +""" from __future__ import annotations +import csv +import json +from dataclasses import dataclass, field +from pathlib import Path + import typer +from rapidfuzz import fuzz +from bggpipe.bgg_client import BGGClient from bggpipe.config import Config +from bggpipe.models import GameVersion +from bggpipe.normalize import normalize_title + +FUZZY_THRESHOLD = 90 +# Tie-break dominance: an exact-named candidate wins outright only if it is +# clearly the well-known game (spec: obscure duplicates lose to famous ones). +DOMINANCE_MIN_OWNED = 100 +DOMINANCE_FACTOR = 10 +VERSION_PLAUSIBLE_SCORE = 2 + +MATCH_COLUMNS = [ + "title_raw", + "bgg_id", + "bgg_name", + "year", + "type", + "match_status", + "version_id", + "version_name", + "version_status", + "candidates_json", + "version_candidates_json", + "source_photos", +] -def run_resolve(cfg: Config, *, force: bool = False) -> None: - typer.echo("bggpipe resolve: not implemented yet (build-order step 2).") - raise typer.Exit(code=1) +@dataclass(frozen=True) +class TitleEntry: + title_raw: str + title_normalized: str + confidence: str = "high" + publisher_hint: str = "" + edition_hint: str = "" + year_hint: int | None = None + language_hint: str = "" + art_notes: str = "" + source_photos: tuple[str, ...] = () + + @property + def has_version_cues(self) -> bool: + return bool( + self.publisher_hint + or self.edition_hint + or self.year_hint + or self.language_hint + ) + + +@dataclass +class Candidate: + bgg_id: int + name: str + year: int | None + type: str + exact: bool + fuzzy: float + owned: int | None = None + rank: int | None = None + + def as_json(self) -> dict: + return { + "bgg_id": self.bgg_id, + "name": self.name, + "year": self.year, + "type": self.type, + "exact": self.exact, + "fuzzy": round(self.fuzzy, 1), + "owned": self.owned, + "rank": self.rank, + } + + +@dataclass +class MatchRow: + title_raw: str + source_photos: tuple[str, ...] = () + bgg_id: int | None = None + bgg_name: str = "" + year: int | None = None + type: str = "" + match_status: str = "unmatched" + version_id: int | None = None + version_name: str = "" + version_status: str = "" + candidates: list[Candidate] = field(default_factory=list) + version_candidates: list[dict] = field(default_factory=list) + + def to_csv(self) -> dict[str, str]: + return { + "title_raw": self.title_raw, + "bgg_id": str(self.bgg_id) if self.bgg_id else "", + "bgg_name": self.bgg_name, + "year": str(self.year) if self.year else "", + "type": self.type, + "match_status": self.match_status, + "version_id": str(self.version_id) if self.version_id else "", + "version_name": self.version_name, + "version_status": self.version_status, + "candidates_json": json.dumps( + [c.as_json() for c in self.candidates], ensure_ascii=False + ), + "version_candidates_json": json.dumps( + self.version_candidates, ensure_ascii=False + ), + "source_photos": ";".join(self.source_photos), + } + + +def load_titles(path: Path) -> list[TitleEntry]: + if not path.exists(): + raise FileNotFoundError( + f"{path} not found — run `bggpipe extract` or hand-write a title list." + ) + entries = [] + for raw in json.loads(path.read_text()): + title_raw = raw["title_raw"] + entries.append( + TitleEntry( + title_raw=title_raw, + title_normalized=raw.get("title_normalized") + or normalize_title(title_raw), + confidence=raw.get("confidence", "high"), + publisher_hint=raw.get("publisher_hint") or "", + edition_hint=raw.get("edition_hint") or "", + year_hint=raw.get("year_hint"), + language_hint=raw.get("language_hint") or "", + art_notes=raw.get("art_notes") or "", + source_photos=tuple(raw.get("source_photos") or ()), + ) + ) + return entries + + +def _plausible_candidates(client: BGGClient, entry: TitleEntry) -> list[Candidate]: + """Search BGG and keep exact-normalized or fuzzy>=90 candidates, one per id.""" + by_id: dict[int, Candidate] = {} + for result in client.search(entry.title_raw): + norm = normalize_title(result.name) + exact = norm == entry.title_normalized + fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized) + if not exact and fuzzy < FUZZY_THRESHOLD: + continue + candidate = Candidate( + bgg_id=result.bgg_id, + name=result.name, + year=result.year, + type=result.type, + exact=exact, + fuzzy=fuzzy, + ) + prev = by_id.get(result.bgg_id) + if prev is None or (candidate.exact, candidate.fuzzy) > ( + prev.exact, + prev.fuzzy, + ): + by_id[result.bgg_id] = candidate + return sorted(by_id.values(), key=lambda c: (not c.exact, -c.fuzzy)) + + +def _classify(client: BGGClient, entry: TitleEntry, cands: list[Candidate]) -> MatchRow: + row = MatchRow(title_raw=entry.title_raw, source_photos=entry.source_photos) + if not cands: + row.match_status = "unmatched" + return row + + if len(cands) == 1: + chosen = cands[0] + else: + top = cands[:5] + stats = { + t.bgg_id: t for t in client.things([c.bgg_id for c in top], stats=True) + } + for c in top: + if c.bgg_id in stats: + c.owned = stats[c.bgg_id].owned + c.rank = stats[c.bgg_id].rank + row.candidates = top + chosen = _dominant(top) + if chosen is None: + row.match_status = "ambiguous" + return row + + row.bgg_id = chosen.bgg_id + row.bgg_name = chosen.name + row.year = chosen.year + row.type = chosen.type + row.match_status = "auto" + row.candidates = row.candidates or [chosen] + return row + + +def _dominant(top: list[Candidate]) -> Candidate | None: + """The single clear winner among plausible candidates, if any. + + Mixed boardgame/expansion candidates are never auto-resolved (the + spec's most common failure mode); otherwise an exact-named candidate + wins only when its owned-count dwarfs the runner-up's. + """ + if len({c.type for c in top}) > 1: + return None + ranked = sorted(top, key=lambda c: c.owned or 0, reverse=True) + best, second = ranked[0], ranked[1] + if ( + best.exact + and (best.owned or 0) >= DOMINANCE_MIN_OWNED + and (best.owned or 0) >= DOMINANCE_FACTOR * max(second.owned or 0, 1) + ): + return best + return None + + +def _score_version(entry: TitleEntry, version: GameVersion) -> int: + score = 0 + if entry.publisher_hint: + hint = normalize_title(entry.publisher_hint) + if any( + fuzz.partial_ratio(hint, normalize_title(p)) >= 85 + for p in version.publishers + ): + score += 2 + if entry.year_hint and version.year == entry.year_hint: + score += 2 + if entry.language_hint and entry.language_hint.casefold() in { + lang.casefold() for lang in version.languages + }: + score += 1 + if ( + entry.edition_hint + and version.name + and fuzz.token_set_ratio( + normalize_title(entry.edition_hint), normalize_title(version.name) + ) + >= 80 + ): + score += 2 + return score + + +def _resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None: + """Fill version_* fields on an auto/approved row. Never guess (spec).""" + if not entry.has_version_cues: + row.version_status = "version_unknown" + return + (thing,) = client.things([row.bgg_id], versions=True) + scored = sorted( + ((v, _score_version(entry, v)) for v in thing.versions), + key=lambda pair: -pair[1], + ) + plausible = [(v, s) for v, s in scored if s >= VERSION_PLAUSIBLE_SCORE] + if not plausible: + row.version_status = "version_unknown" + return + row.version_candidates = [ + { + "version_id": v.version_id, + "name": v.name, + "year": v.year, + "publishers": list(v.publishers), + "languages": list(v.languages), + "score": s, + } + for v, s in plausible[:8] + ] + if len(plausible) == 1 or plausible[0][1] > plausible[1][1]: + winner = plausible[0][0] + row.version_id = winner.version_id + row.version_name = winner.name + row.version_status = "version_auto" + else: + row.version_status = "version_ambiguous" + + +def resolve_entry(client: BGGClient, entry: TitleEntry) -> MatchRow: + row = _classify(client, entry, _plausible_candidates(client, entry)) + if row.match_status == "auto": + _resolve_version(client, entry, row) + return row + + +def _row_key(title_raw: str, source_photos: str) -> tuple[str, str]: + return (title_raw, source_photos) + + +def read_existing_keys(path: Path) -> set[tuple[str, str]]: + if not path.exists(): + return set() + with path.open(newline="") as f: + return {_row_key(r["title_raw"], r["source_photos"]) for r in csv.DictReader(f)} + + +def append_rows(path: Path, rows: list[MatchRow]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + is_new = not path.exists() + with path.open("a", newline="") as f: + writer = csv.DictWriter(f, fieldnames=MATCH_COLUMNS) + if is_new: + writer.writeheader() + for row in rows: + writer.writerow(row.to_csv()) + + +def run_resolve( + cfg: Config, *, force: bool = False, client: BGGClient | None = None +) -> list[MatchRow]: + entries = load_titles(cfg.titles_path) + if force and cfg.matches_path.exists(): + cfg.matches_path.unlink() + existing = read_existing_keys(cfg.matches_path) + client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds) + + new_rows: list[MatchRow] = [] + skipped = 0 + for entry in entries: + key = _row_key(entry.title_raw, ";".join(entry.source_photos)) + if key in existing: + skipped += 1 + continue + row = resolve_entry(client, entry) + new_rows.append(row) + detail = f"{row.bgg_name} ({row.bgg_id})" if row.bgg_id else "-" + version = f" [{row.version_status}]" if row.version_status else "" + typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}") + + append_rows(cfg.matches_path, new_rows) + + counts: dict[str, int] = {} + for row in new_rows: + counts[row.match_status] = counts.get(row.match_status, 0) + 1 + summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items())) + typer.echo( + f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); " + f"skipped {skipped} already in {cfg.matches_path}." + ) + return new_rows diff --git a/tests/fixtures/bgg_cache/search_query=Blorvath-3A-Quest-of-the-Zzyzx-type=boardgame-2Cboardgameexpansion_22856aa961.xml b/tests/fixtures/bgg_cache/search_query=Blorvath-3A-Quest-of-the-Zzyzx-type=boardgame-2Cboardgameexpansion_22856aa961.xml new file mode 100644 index 0000000..ba791c3 --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Blorvath-3A-Quest-of-the-Zzyzx-type=boardgame-2Cboardgameexpansion_22856aa961.xml @@ -0,0 +1 @@ +<items total="0"></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/search_query=Caf-C3-A9-International-type=boardgame-2Cboardgameexpansion_d875abc790.xml b/tests/fixtures/bgg_cache/search_query=Caf-C3-A9-International-type=boardgame-2Cboardgameexpansion_d875abc790.xml new file mode 100644 index 0000000..3cb54f8 --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Caf-C3-A9-International-type=boardgame-2Cboardgameexpansion_d875abc790.xml @@ -0,0 +1 @@ +<items total="2"><item type="boardgame" id="373"><name type="primary" value="Café International"/><yearpublished value="1989"/></item><item type="boardgame" id="8103"><name type="primary" value="Café International: Das Kartenspiel"/><yearpublished value="2001"/></item></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/search_query=Catan-type=boardgame-2Cboardgameexpansion_5be385814d.xml b/tests/fixtures/bgg_cache/search_query=Catan-type=boardgame-2Cboardgameexpansion_5be385814d.xml new file mode 100644 index 0000000..7343b91 --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Catan-type=boardgame-2Cboardgameexpansion_5be385814d.xml @@ -0,0 +1 @@ +<items total="4"><item type="boardgame" id="13"><name type="primary" value="CATAN"/><yearpublished value="1995"/></item><item type="boardgameexpansion" id="926"><name type="primary" value="CATAN: Seafarers"/><yearpublished value="1997"/></item><item type="boardgame" id="27760"><name type="primary" value="Catan Dice Game"/><yearpublished value="2007"/></item><item type="boardgame" id="133038"><name type="primary" value="CATAN 3D Collector's Edition"/><yearpublished value="2005"/></item></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/search_query=Citadels-type=boardgame-2Cboardgameexpansion_9f48c481c4.xml b/tests/fixtures/bgg_cache/search_query=Citadels-type=boardgame-2Cboardgameexpansion_9f48c481c4.xml new file mode 100644 index 0000000..e76b16c --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Citadels-type=boardgame-2Cboardgameexpansion_9f48c481c4.xml @@ -0,0 +1 @@ +<items total="2"><item type="boardgame" id="478"><name type="primary" value="Citadels"/><yearpublished value="2000"/></item><item type="boardgame" id="205398"><name type="primary" value="Citadels"/><yearpublished value="2016"/></item></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/search_query=Wingspan-3A-European-Expansion-type=boardgame-2Cboardgameexpansion_85d6b5d538.xml b/tests/fixtures/bgg_cache/search_query=Wingspan-3A-European-Expansion-type=boardgame-2Cboardgameexpansion_85d6b5d538.xml new file mode 100644 index 0000000..e99a25e --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Wingspan-3A-European-Expansion-type=boardgame-2Cboardgameexpansion_85d6b5d538.xml @@ -0,0 +1 @@ +<items total="4"><item type="boardgame" id="266192"><name type="primary" value="Wingspan"/><yearpublished value="2019"/></item><item type="boardgameexpansion" id="290448"><name type="primary" value="Wingspan: European Expansion"/><yearpublished value="2019"/></item><item type="boardgameexpansion" id="300580"><name type="primary" value="Wingspan: Oceania Expansion"/><yearpublished value="2020"/></item><item type="boardgame" id="366161"><name type="primary" value="Wingspan Asia"/><yearpublished value="2022"/></item></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/search_query=Wingspan-Europe-type=boardgame-2Cboardgameexpansion_f5cb460ffe.xml b/tests/fixtures/bgg_cache/search_query=Wingspan-Europe-type=boardgame-2Cboardgameexpansion_f5cb460ffe.xml new file mode 100644 index 0000000..e99a25e --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Wingspan-Europe-type=boardgame-2Cboardgameexpansion_f5cb460ffe.xml @@ -0,0 +1 @@ +<items total="4"><item type="boardgame" id="266192"><name type="primary" value="Wingspan"/><yearpublished value="2019"/></item><item type="boardgameexpansion" id="290448"><name type="primary" value="Wingspan: European Expansion"/><yearpublished value="2019"/></item><item type="boardgameexpansion" id="300580"><name type="primary" value="Wingspan: Oceania Expansion"/><yearpublished value="2020"/></item><item type="boardgame" id="366161"><name type="primary" value="Wingspan Asia"/><yearpublished value="2022"/></item></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/search_query=Wingspan-type=boardgame-2Cboardgameexpansion_0316afc400.xml b/tests/fixtures/bgg_cache/search_query=Wingspan-type=boardgame-2Cboardgameexpansion_0316afc400.xml new file mode 100644 index 0000000..e99a25e --- /dev/null +++ b/tests/fixtures/bgg_cache/search_query=Wingspan-type=boardgame-2Cboardgameexpansion_0316afc400.xml @@ -0,0 +1 @@ +<items total="4"><item type="boardgame" id="266192"><name type="primary" value="Wingspan"/><yearpublished value="2019"/></item><item type="boardgameexpansion" id="290448"><name type="primary" value="Wingspan: European Expansion"/><yearpublished value="2019"/></item><item type="boardgameexpansion" id="300580"><name type="primary" value="Wingspan: Oceania Expansion"/><yearpublished value="2020"/></item><item type="boardgame" id="366161"><name type="primary" value="Wingspan Asia"/><yearpublished value="2022"/></item></items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/thing_id=266192-versions=1_4904563f7f.xml b/tests/fixtures/bgg_cache/thing_id=266192-versions=1_4904563f7f.xml new file mode 100644 index 0000000..22f21a2 --- /dev/null +++ b/tests/fixtures/bgg_cache/thing_id=266192-versions=1_4904563f7f.xml @@ -0,0 +1,25 @@ +<items> + <item type="boardgame" id="266192"> + <name type="primary" value="Wingspan"/><yearpublished value="2019"/> + <versions> + <item type="boardgameversion" id="465063"> + <name type="primary" value="English first edition"/> + <yearpublished value="2019"/> + <link type="boardgamepublisher" id="23202" value="Stonemaier Games"/> + <link type="language" id="2184" value="English"/> + </item> + <item type="boardgameversion" id="521212"> + <name type="primary" value="English fourth printing"/> + <yearpublished value="2020"/> + <link type="boardgamepublisher" id="23202" value="Stonemaier Games"/> + <link type="language" id="2184" value="English"/> + </item> + <item type="boardgameversion" id="472430"> + <name type="primary" value="German edition"/> + <yearpublished value="2019"/> + <link type="boardgamepublisher" id="22160" value="Feuerland Spiele"/> + <link type="language" id="2188" value="German"/> + </item> + </versions> + </item> +</items> \ No newline at end of file diff --git a/tests/fixtures/bgg_cache/thing_id=478-2C205398-stats=1_9bd49e518b.xml b/tests/fixtures/bgg_cache/thing_id=478-2C205398-stats=1_9bd49e518b.xml new file mode 100644 index 0000000..bbc07d7 --- /dev/null +++ b/tests/fixtures/bgg_cache/thing_id=478-2C205398-stats=1_9bd49e518b.xml @@ -0,0 +1,14 @@ +<items> + <item type="boardgame" id="478"> + <name type="primary" value="Citadels"/><yearpublished value="2000"/> + <statistics><ratings><owned value="85000"/> + <ranks><rank type="subtype" id="1" name="boardgame" value="250"/></ranks> + </ratings></statistics> + </item> + <item type="boardgame" id="205398"> + <name type="primary" value="Citadels"/><yearpublished value="2016"/> + <statistics><ratings><owned value="24000"/> + <ranks><rank type="subtype" id="1" name="boardgame" value="400"/></ranks> + </ratings></statistics> + </item> +</items> \ No newline at end of file diff --git a/tests/test_client.py b/tests/test_client.py index 1e9d1a5..79ea3e3 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -108,3 +108,26 @@ def test_cache_key_stable_and_filename_safe(tmp_path): key2 = cache_key("search", {"type": "boardgame", "query": "Café & Krieg?"}) assert key1 == key2 # param order must not matter assert "/" not in key1 and "?" not in key1 and key1.endswith(".xml") + + +def test_api_token_sent_as_bearer_header(tmp_path, monkeypatch): + monkeypatch.setenv("BGG_API_TOKEN", "test-token-123") + client, calls, _ = make_client(tmp_path, [(200, SEARCH_XML)]) + client.get_xml("search", {"query": "catan"}) + assert calls[0].headers["Authorization"] == "Bearer test-token-123" + + +def test_no_auth_header_without_token(tmp_path, monkeypatch): + monkeypatch.delenv("BGG_API_TOKEN", raising=False) + client, calls, _ = make_client(tmp_path, [(200, SEARCH_XML)]) + client.get_xml("search", {"query": "catan"}) + assert "authorization" not in calls[0].headers + + +def test_401_raises_actionable_auth_error(tmp_path, monkeypatch): + monkeypatch.delenv("BGG_API_TOKEN", raising=False) + from bggpipe.bgg_client import BGGAuthError + + client, _, _ = make_client(tmp_path, [(401, "Unauthorized")]) + with pytest.raises(BGGAuthError, match="BGG_API_TOKEN"): + client.get_xml("search", {"query": "catan"}) diff --git a/tests/test_resolve.py b/tests/test_resolve.py new file mode 100644 index 0000000..1cd9859 --- /dev/null +++ b/tests/test_resolve.py @@ -0,0 +1,189 @@ +"""Resolve-stage tests, replayed from tests/fixtures/bgg_cache — never online. + +The transport below raises on any network attempt, proving every response +comes from the committed fixture cache. +""" + +from __future__ import annotations + +import csv +import json +import shutil +from pathlib import Path + +import httpx +import pytest + +from bggpipe.bgg_client import BGGClient +from bggpipe.config import Config +from bggpipe.models import GameVersion +from bggpipe.resolve import ( + Candidate, + TitleEntry, + _dominant, + _score_version, + load_titles, + resolve_entry, + run_resolve, +) + +FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache" +TITLES_JSON = Path(__file__).parent.parent / "data" / "titles.json" + + +def _no_network(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"test hit the network: {request.url}") + + +@pytest.fixture +def client() -> BGGClient: + return BGGClient( + cache_dir=FIXTURES, + transport=httpx.MockTransport(_no_network), + sleep=lambda s: pytest.fail("slept during fixture replay"), + ) + + +@pytest.fixture +def rows_by_title(client) -> dict[str, object]: + entries = load_titles(TITLES_JSON) + return {e.title_raw: resolve_entry(client, e) for e in entries} + + +def test_catan_auto_matches_base_game(rows_by_title): + row = rows_by_title["Catan"] + assert (row.match_status, row.bgg_id, row.bgg_name) == ("auto", 13, "CATAN") + assert row.version_status == "version_unknown" # no cues -> never guess + + +def test_wingspan_auto_with_version_from_cues(rows_by_title): + row = rows_by_title["Wingspan"] + assert (row.match_status, row.bgg_id) == ("auto", 266192) + # cues: Stonemaier + 2019 + English -> the first edition, not the + # 2020 printing (score 5 vs 3) and not the German edition + assert row.version_status == "version_auto" + assert row.version_id == 465063 + assert row.version_name == "English first edition" + + +def test_expansion_matches_expansion_not_base(rows_by_title): + row = rows_by_title["Wingspan: European Expansion"] + assert (row.match_status, row.bgg_id) == ("auto", 290448) + assert row.type == "boardgameexpansion" + + +def test_wingspan_europe_spine_never_matches_base(rows_by_title): + """The spec's canonical trap: a truncated spine must not auto-match.""" + row = rows_by_title["Wingspan Europe"] + assert row.match_status == "unmatched" + assert row.bgg_id is None + + +def test_accented_title_resolves(rows_by_title): + row = rows_by_title["Café International"] + assert (row.match_status, row.bgg_id) == ("auto", 373) + + +def test_same_name_editions_stay_ambiguous(rows_by_title): + row = rows_by_title["Citadels"] + assert row.match_status == "ambiguous" + assert {c.bgg_id for c in row.candidates} == {478, 205398} + # stats were fetched for the review UI + assert all(c.owned is not None for c in row.candidates) + + +def test_nonsense_lands_unmatched(rows_by_title): + row = rows_by_title["Blorvath: Quest of the Zzyzx"] + assert row.match_status == "unmatched" + + +def test_run_resolve_writes_csv_and_is_idempotent(client, tmp_path): + data_dir = tmp_path / "data" + data_dir.mkdir() + shutil.copy(TITLES_JSON, data_dir / "titles.json") + cfg = Config(data_dir=data_dir) + + first = run_resolve(cfg, client=client) + assert len(first) == 7 + + with cfg.matches_path.open(newline="") as f: + rows = list(csv.DictReader(f)) + assert len(rows) == 7 + by_title = {r["title_raw"]: r for r in rows} + assert by_title["Catan"]["match_status"] == "auto" + assert by_title["Citadels"]["match_status"] == "ambiguous" + candidates = json.loads(by_title["Citadels"]["candidates_json"]) + assert len(candidates) == 2 + assert by_title["Wingspan"]["version_id"] == "465063" + + # second run: everything already in matches.csv is skipped, file unchanged + before = cfg.matches_path.read_text() + second = run_resolve(cfg, client=client) + assert second == [] + assert cfg.matches_path.read_text() == before + + +# -- pure-logic unit tests (no fixtures) -------------------------------- + + +def _cand(bgg_id, exact=True, owned=None, type_="boardgame"): + return Candidate( + bgg_id=bgg_id, + name="X", + year=2000, + type=type_, + exact=exact, + fuzzy=100.0, + owned=owned, + ) + + +def test_dominant_famous_game_beats_obscure_duplicate(): + top = [_cand(1, owned=50000), _cand(2, owned=300)] + assert _dominant(top).bgg_id == 1 + + +def test_dominant_refuses_mixed_types(): + top = [_cand(1, owned=50000), _cand(2, owned=300, type_="boardgameexpansion")] + assert _dominant(top) is None + + +def test_dominant_refuses_close_call(): + top = [_cand(1, owned=50000), _cand(2, owned=20000)] + assert _dominant(top) is None + + +def test_score_version_all_cues(): + entry = TitleEntry( + title_raw="Wingspan", + title_normalized="wingspan", + publisher_hint="Stonemaier", + year_hint=2019, + language_hint="english", + ) + version = GameVersion( + version_id=1, + name="English first edition", + year=2019, + publishers=("Stonemaier Games",), + languages=("English",), + ) + assert _score_version(entry, version) == 5 + + +def test_score_version_no_overlap(): + entry = TitleEntry( + title_raw="Wingspan", + title_normalized="wingspan", + publisher_hint="Feuerland", + year_hint=2021, + language_hint="german", + ) + version = GameVersion( + version_id=1, + name="English first edition", + year=2019, + publishers=("Stonemaier Games",), + languages=("English",), + ) + assert _score_version(entry, version) == 0 diff --git a/tests/test_smoke_live.py b/tests/test_smoke_live.py new file mode 100644 index 0000000..8f7ccca --- /dev/null +++ b/tests/test_smoke_live.py @@ -0,0 +1,31 @@ +"""The one test allowed to touch the real BGG API. Read-only, skipped by +default; run with: uv run pytest --run-live -m live + +Needs bgg_username in config.toml and BGG_API_TOKEN in the environment +(register at https://boardgamegeek.com/applications). +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from bggpipe.bgg_client import BGGClient +from bggpipe.config import load_config + + +@pytest.mark.live +def test_fetch_own_collection_read_only(tmp_path: Path) -> None: + cfg = load_config() + if not cfg.bgg_username: + pytest.skip("set bgg_username in config.toml to run the live smoke test") + if not os.environ.get("BGG_API_TOKEN"): + pytest.skip("set BGG_API_TOKEN to run the live smoke test") + + # fresh cache dir so this genuinely exercises the live API + 202 queue + client = BGGClient(cache_dir=tmp_path / "cache") + items = client.collection_full(cfg.bgg_username) + assert isinstance(items, list) + assert all(item.own for item in items)