From 4af83d362661f032154a60b0f2014b2e70871f54 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Sat, 1 Aug 2026 14:37:56 -0400 Subject: [PATCH] Enrich stage: full game + version metadata into games.json MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Batched /thing?stats=1 fetches (20 ids, sorted so batch cache keys stay stable), parsing the full frontend-seed payload: designers, artists, publishers, player counts with Best-majority poll analysis, playtimes, min age, weight, rating, rank, categories, mechanics, description, and image URLs. Chosen-version details are reused from matches.csv's stored candidates — zero extra API calls. Already-enriched keys are skipped entirely; --refresh bypasses the cache read since ranks and ratings drift. Degrades gracefully without BGG_API_TOKEN: cached ids enrich, the rest wait, everything fetched is saved. Co-Authored-By: Claude Fable 5 --- src/bggpipe/bgg_client.py | 16 ++- src/bggpipe/cli.py | 5 +- src/bggpipe/enrich.py | 107 ++++++++++++++++++ src/bggpipe/models.py | 74 +++++++++++++ tests/test_enrich.py | 228 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 426 insertions(+), 4 deletions(-) create mode 100644 src/bggpipe/enrich.py create mode 100644 tests/test_enrich.py diff --git a/src/bggpipe/bgg_client.py b/src/bggpipe/bgg_client.py index e148ffe..60257cf 100644 --- a/src/bggpipe/bgg_client.py +++ b/src/bggpipe/bgg_client.py @@ -25,6 +25,7 @@ from bggpipe.models import ( parse_collection, parse_search, parse_things, + parse_things_full, ) BASE_URL = "https://boardgamegeek.com/xmlapi2" @@ -83,10 +84,14 @@ class BGGClient: if wait > 0: self._sleep(wait) - def get_xml(self, endpoint: str, params: dict[str, str]) -> str: - """Fetch one endpoint, serving from and filling the disk cache.""" + def get_xml( + self, endpoint: str, params: dict[str, str], *, refresh: bool = False + ) -> str: + """Fetch one endpoint, serving from and filling the disk cache. + refresh=True skips the cache read (still writes) — for data that + drifts over time, like ranks and ratings.""" cache_path = self.cache_dir / cache_key(endpoint, params) - if cache_path.exists(): + if cache_path.exists() and not refresh: return cache_path.read_text() for attempt in range(MAX_ATTEMPTS): @@ -142,6 +147,11 @@ class BGGClient: params["versions"] = "1" return parse_things(self.get_xml("thing", params)) + def things_full(self, ids: Iterable[int], *, refresh: bool = False) -> list[dict]: + """Full metadata dicts for the enrich stage.""" + params = {"id": ",".join(str(i) for i in ids), "stats": "1"} + return parse_things_full(self.get_xml("thing", params, refresh=refresh)) + def collection( self, username: str, diff --git a/src/bggpipe/cli.py b/src/bggpipe/cli.py index e322025..54d23ae 100644 --- a/src/bggpipe/cli.py +++ b/src/bggpipe/cli.py @@ -93,4 +93,7 @@ def enrich( config: ConfigOpt = None, ) -> None: """Stage 6: fetch full game + version metadata into games.json.""" - _not_implemented("enrich", 6) + from bggpipe.enrich import run_enrich + + cfg = load_config(config) + run_enrich(cfg, refresh=refresh) diff --git a/src/bggpipe/enrich.py b/src/bggpipe/enrich.py new file mode 100644 index 0000000..449f557 --- /dev/null +++ b/src/bggpipe/enrich.py @@ -0,0 +1,107 @@ +"""Stage 6 — enrich: full game + version metadata into games.json. + +The seed data for the future web frontend, keyed by bgg_id (or +"bgg_id:version_id" when a version is settled). Cheap by design: +- ids are batched (~20 per /thing call) and sorted so batch cache keys + stay stable across runs; +- already-enriched keys are skipped entirely (no request, no cache read) + unless --refresh, which bypasses the cache read because ranks and + ratings drift over time; +- the chosen version's details come from matches.csv's stored version + candidates — no extra API calls. +Degrades gracefully without BGG_API_TOKEN: whatever is cached enriches, +the rest waits for the token. +""" + +from __future__ import annotations + +import json + +import typer + +from bggpipe.bgg_client import BGGAuthError, BGGClient +from bggpipe.config import Config +from bggpipe.resolve import read_matches + +BATCH_SIZE = 20 +_CONFIDENT_VERSION = ("version_auto", "version_approved") + + +def _version_info(row: dict) -> dict | None: + if row["version_status"] not in _CONFIDENT_VERSION or not row["version_id"]: + return None + version_id = int(row["version_id"]) + for cand in json.loads(row["version_candidates_json"] or "[]"): + if cand.get("version_id") == version_id: + return { + "version_id": version_id, + "name": cand.get("name", ""), + "year": cand.get("year"), + "publishers": cand.get("publishers") or [], + "languages": cand.get("languages") or [], + } + # candidates were pruned (e.g. manual review) — keep what the row knows + return { + "version_id": version_id, + "name": row["version_name"], + "year": None, + "publishers": [], + "languages": [], + } + + +def run_enrich( + cfg: Config, *, refresh: bool = False, client: BGGClient | None = None +) -> dict: + rows = read_matches(cfg.matches_path) + if not rows: + typer.echo(f"{cfg.matches_path} is empty — run `bggpipe resolve` first.") + raise typer.Exit(code=1) + + targets: list[tuple[str, int, dict | None]] = [] + for row in rows: + if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]: + continue + version = _version_info(row) + key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"] + targets.append((key, int(row["bgg_id"]), version)) + + games_path = cfg.data_dir / "games.json" + games: dict = json.loads(games_path.read_text()) if games_path.exists() else {} + + need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games}) + client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds) + + fetched: dict[int, dict] = {} + blocked = False + for start in range(0, len(need), BATCH_SIZE): + batch = need[start : start + BATCH_SIZE] + try: + for thing in client.things_full(batch, refresh=refresh): + fetched[thing["bgg_id"]] = thing + except BGGAuthError: + blocked = True + break + + updated = 0 + for key, bgg_id, version in targets: + if bgg_id in fetched: + games[key] = {**fetched[bgg_id], "version": version} + updated += 1 + + games_path.parent.mkdir(parents=True, exist_ok=True) + games_path.write_text(json.dumps(games, indent=2, ensure_ascii=False) + "\n") + + typer.echo( + f"games.json: {len(games)} entr{'y' if len(games) == 1 else 'ies'} " + f"({updated} added/refreshed this run; " + f"{len(targets) - updated} already present or waiting)." + ) + if blocked: + remaining = [i for i in need if i not in fetched] + typer.echo( + f"\n{len(remaining)} game(s) are waiting on the BGG API " + "(set BGG_API_TOKEN and re-run enrich — everything fetched " + "so far is saved)." + ) + return games diff --git a/src/bggpipe/models.py b/src/bggpipe/models.py index bd2f64a..9c96bc0 100644 --- a/src/bggpipe/models.py +++ b/src/bggpipe/models.py @@ -135,6 +135,80 @@ def parse_things(xml_text: str) -> list[ThingDetails]: return things +def _attr_float(elem: ET.Element | None, attr: str = "value") -> float | None: + if elem is None: + return None + try: + return float(elem.get(attr)) + except (TypeError, ValueError): + return None + + +def parse_things_full(xml_text: str) -> list[dict]: + """Full game metadata for the enrich stage (games.json). Returns plain + dicts — this is artifact data, not pipeline logic.""" + games = [] + for item in _root(xml_text).findall("item"): + + def links(link_type: str, item: ET.Element = item) -> list[str]: + return [ + link.get("value", "") + for link in item.findall(f"link[@type='{link_type}']") + ] + + name = item.find("name[@type='primary']") + ratings = item.find("statistics/ratings") + best_player_counts = [] + poll = item.find("poll[@name='suggested_numplayers']") + if poll is not None: + for results in poll.findall("results"): + votes = { + r.get("value"): int(r.get("numvotes") or 0) + for r in results.findall("result") + } + best = votes.get("Best", 0) + if ( + best + and best >= votes.get("Recommended", 0) + and best > votes.get("Not Recommended", 0) + ): + best_player_counts.append(results.get("numplayers")) + games.append( + { + "bgg_id": int(item.get("id", 0)), + "type": item.get("type", "boardgame"), + "name": name.get("value", "") if name is not None else "", + "year": _attr_int(item.find("yearpublished")), + "description": (item.findtext("description") or "").strip(), + "image": (item.findtext("image") or "").strip(), + "thumbnail": (item.findtext("thumbnail") or "").strip(), + "min_players": _attr_int(item.find("minplayers")), + "max_players": _attr_int(item.find("maxplayers")), + "best_player_counts": best_player_counts, + "playtime": _attr_int(item.find("playingtime")), + "min_playtime": _attr_int(item.find("minplaytime")), + "max_playtime": _attr_int(item.find("maxplaytime")), + "min_age": _attr_int(item.find("minage")), + "designers": links("boardgamedesigner"), + "artists": links("boardgameartist"), + "publishers": links("boardgamepublisher"), + "categories": links("boardgamecategory"), + "mechanics": links("boardgamemechanic"), + "rating": _attr_float(ratings.find("average")) + if ratings is not None + else None, + "weight": _attr_float(ratings.find("averageweight")) + if ratings is not None + else None, + "rank": _attr_int(item.find(".//ranks/rank[@name='boardgame']")), + "users_owned": _attr_int(ratings.find("owned")) + if ratings is not None + else None, + } + ) + return games + + def parse_collection(xml_text: str) -> list[CollectionItem]: items = [] for item in _root(xml_text).findall("item"): diff --git a/tests/test_enrich.py b/tests/test_enrich.py new file mode 100644 index 0000000..a34c0d4 --- /dev/null +++ b/tests/test_enrich.py @@ -0,0 +1,228 @@ +"""Enrich-stage tests: full-metadata parsing plus run_enrich orchestration +(batched, cache-keyed, idempotent, token-degrading). No network.""" + +from __future__ import annotations + +import json + +import httpx + +from bggpipe.bgg_client import BGGClient, cache_key +from bggpipe.config import Config +from bggpipe.models import parse_things_full +from bggpipe.resolve import write_matches + +FULL_THING_XML = """ + + https://cf.example/thumb.jpg + https://cf.example/full.jpg + + + A bird-collection engine builder. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +def test_parse_things_full_extracts_everything(): + (game,) = parse_things_full(FULL_THING_XML) + assert game["name"] == "Wingspan" + assert game["year"] == 2019 + assert game["description"] == "A bird-collection engine builder." + assert (game["min_players"], game["max_players"]) == (1, 5) + # 3 is Best-majority; 4 loses to Recommended+NotRec? no — Best(90) >= Rec(80) + # and > NotRec(5), so 4 qualifies too; 1 does not (Best < Recommended) + assert game["best_player_counts"] == ["3", "4"] + assert game["designers"] == ["Elizabeth Hargrave"] + assert game["artists"] == ["Natalia Rojas"] + assert game["publishers"] == ["Stonemaier Games"] + assert game["categories"] == ["Animals", "Card Game"] + assert game["mechanics"] == ["Engine Building"] + assert game["rating"] == 8.05 + assert game["weight"] == 2.45 + assert game["rank"] == 30 + assert game["playtime"] == 70 + assert game["min_age"] == 10 + assert game["image"].endswith("full.jpg") + + +CATAN_MINIMAL = ( + '' + '' + "" +) + + +def _matches_rows(): + base = { + "year": "", + "type": "boardgame", + "candidates_json": "[]", + "source_photos": "x.jpg", + "version_id": "", + "version_name": "", + "version_status": "version_unknown", + "version_candidates_json": "[]", + } + return [ + { + **base, + "title_raw": "Catan", + "bgg_id": "13", + "bgg_name": "CATAN", + "match_status": "auto", + }, + { + **base, + "title_raw": "Wingspan", + "bgg_id": "266192", + "bgg_name": "Wingspan", + "match_status": "auto", + "version_id": "465063", + "version_name": "English first edition", + "version_status": "version_auto", + "version_candidates_json": json.dumps( + [ + { + "version_id": 465063, + "name": "English first edition", + "year": 2019, + "publishers": ["Stonemaier Games"], + "languages": ["English"], + "score": 5, + } + ] + ), + }, + { + **base, + "title_raw": "Junk", + "bgg_id": "", + "bgg_name": "", + "match_status": "rejected", + }, + ] + + +def _no_network(request: httpx.Request) -> httpx.Response: + raise AssertionError(f"test hit the network: {request.url}") + + +def _cfg_with_matches(tmp_path): + cfg = Config(data_dir=tmp_path / "data") + write_matches(cfg.matches_path, _matches_rows()) + return cfg + + +def _seed_batch_fixture(cache_dir): + """The batch cache entry run_enrich will ask for: sorted ids 13,266192.""" + cache_dir.mkdir(parents=True, exist_ok=True) + combined = FULL_THING_XML.replace("", "" + CATAN_MINIMAL[7:-8], 1) + key = cache_key("thing", {"id": "13,266192", "stats": "1"}) + (cache_dir / key).write_text(combined) + + +def test_run_enrich_writes_games_json_with_versions(tmp_path): + from bggpipe.enrich import run_enrich + + cfg = _cfg_with_matches(tmp_path) + cache = tmp_path / "cache" + _seed_batch_fixture(cache) + client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network)) + + games = run_enrich(cfg, client=client) + assert set(games) == {"13", "266192:465063"} + wingspan = games["266192:465063"] + assert wingspan["designers"] == ["Elizabeth Hargrave"] + assert wingspan["version"]["name"] == "English first edition" + assert wingspan["version"]["publishers"] == ["Stonemaier Games"] + assert games["13"]["version"] is None + saved = json.loads((cfg.data_dir / "games.json").read_text()) + assert saved == games + + +def test_run_enrich_skips_already_enriched(tmp_path): + from bggpipe.enrich import run_enrich + + cfg = _cfg_with_matches(tmp_path) + cache = tmp_path / "cache" + _seed_batch_fixture(cache) + client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network)) + run_enrich(cfg, client=client) + + # empty cache + network-refusing transport: passes only if enrich + # doesn't need to fetch anything at all + bare = BGGClient( + cache_dir=tmp_path / "empty", transport=httpx.MockTransport(_no_network) + ) + games = run_enrich(cfg, client=bare) + assert set(games) == {"13", "266192:465063"} + + +def test_refresh_bypasses_cache_read(tmp_path): + from bggpipe.enrich import run_enrich + + cfg = _cfg_with_matches(tmp_path) + cache = tmp_path / "cache" + _seed_batch_fixture(cache) + requests = [] + + def handler(request): + requests.append(str(request.url)) + return httpx.Response(200, text=FULL_THING_XML.replace("8.05", "7.5")) + + client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(handler)) + games = run_enrich(cfg, refresh=True, client=client) + assert len(requests) == 1 # cache read skipped, live fetch happened + assert games["266192:465063"]["rating"] == 7.5 + + +def test_enrich_degrades_without_token(tmp_path, capsys): + from bggpipe.enrich import run_enrich + + cfg = _cfg_with_matches(tmp_path) + client = BGGClient( + cache_dir=tmp_path / "empty", + transport=httpx.MockTransport( + lambda req: httpx.Response(401, text="Unauthorized") + ), + ) + games = run_enrich(cfg, client=client) + assert games == {} + assert "waiting on the BGG API" in capsys.readouterr().out + assert json.loads((cfg.data_dir / "games.json").read_text()) == {}