Enrich stage: full game + version metadata into games.json

Batched /thing?stats=1 fetches (20 ids, sorted so batch cache keys stay
stable), parsing the full frontend-seed payload: designers, artists,
publishers, player counts with Best-majority poll analysis, playtimes,
min age, weight, rating, rank, categories, mechanics, description, and
image URLs. Chosen-version details are reused from matches.csv's stored
candidates — zero extra API calls. Already-enriched keys are skipped
entirely; --refresh bypasses the cache read since ranks and ratings
drift. Degrades gracefully without BGG_API_TOKEN: cached ids enrich,
the rest wait, everything fetched is saved.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

src/bggpipe/bgg_client.py

@@ -25,6 +25,7 @@ from bggpipe.models import (
     parse_collection,
     parse_search,
     parse_things,
+    parse_things_full,
 )
 
 BASE_URL = "https://boardgamegeek.com/xmlapi2"
@@ -83,10 +84,14 @@ class BGGClient:
             if wait > 0:
                 self._sleep(wait)
 
-    def get_xml(self, endpoint: str, params: dict[str, str]) -> str:
-        """Fetch one endpoint, serving from and filling the disk cache."""
+    def get_xml(
+        self, endpoint: str, params: dict[str, str], *, refresh: bool = False
+    ) -> str:
+        """Fetch one endpoint, serving from and filling the disk cache.
+        refresh=True skips the cache read (still writes) — for data that
+        drifts over time, like ranks and ratings."""
         cache_path = self.cache_dir / cache_key(endpoint, params)
-        if cache_path.exists():
+        if cache_path.exists() and not refresh:
             return cache_path.read_text()
 
         for attempt in range(MAX_ATTEMPTS):
@@ -142,6 +147,11 @@ class BGGClient:
             params["versions"] = "1"
         return parse_things(self.get_xml("thing", params))
 
+    def things_full(self, ids: Iterable[int], *, refresh: bool = False) -> list[dict]:
+        """Full metadata dicts for the enrich stage."""
+        params = {"id": ",".join(str(i) for i in ids), "stats": "1"}
+        return parse_things_full(self.get_xml("thing", params, refresh=refresh))
+
     def collection(
         self,
         username: str,

src/bggpipe/cli.py

@@ -93,4 +93,7 @@ def enrich(
     config: ConfigOpt = None,
 ) -> None:
     """Stage 6: fetch full game + version metadata into games.json."""
-    _not_implemented("enrich", 6)
+    from bggpipe.enrich import run_enrich
+
+    cfg = load_config(config)
+    run_enrich(cfg, refresh=refresh)

src/bggpipe/enrich.py

@@ -0,0 +1,107 @@
+"""Stage 6 — enrich: full game + version metadata into games.json.
+
+The seed data for the future web frontend, keyed by bgg_id (or
+"bgg_id:version_id" when a version is settled). Cheap by design:
+- ids are batched (~20 per /thing call) and sorted so batch cache keys
+  stay stable across runs;
+- already-enriched keys are skipped entirely (no request, no cache read)
+  unless --refresh, which bypasses the cache read because ranks and
+  ratings drift over time;
+- the chosen version's details come from matches.csv's stored version
+  candidates — no extra API calls.
+Degrades gracefully without BGG_API_TOKEN: whatever is cached enriches,
+the rest waits for the token.
+"""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from bggpipe.bgg_client import BGGAuthError, BGGClient
+from bggpipe.config import Config
+from bggpipe.resolve import read_matches
+
+BATCH_SIZE = 20
+_CONFIDENT_VERSION = ("version_auto", "version_approved")
+
+
+def _version_info(row: dict) -> dict | None:
+    if row["version_status"] not in _CONFIDENT_VERSION or not row["version_id"]:
+        return None
+    version_id = int(row["version_id"])
+    for cand in json.loads(row["version_candidates_json"] or "[]"):
+        if cand.get("version_id") == version_id:
+            return {
+                "version_id": version_id,
+                "name": cand.get("name", ""),
+                "year": cand.get("year"),
+                "publishers": cand.get("publishers") or [],
+                "languages": cand.get("languages") or [],
+            }
+    # candidates were pruned (e.g. manual review) — keep what the row knows
+    return {
+        "version_id": version_id,
+        "name": row["version_name"],
+        "year": None,
+        "publishers": [],
+        "languages": [],
+    }
+
+
+def run_enrich(
+    cfg: Config, *, refresh: bool = False, client: BGGClient | None = None
+) -> dict:
+    rows = read_matches(cfg.matches_path)
+    if not rows:
+        typer.echo(f"{cfg.matches_path} is empty — run `bggpipe resolve` first.")
+        raise typer.Exit(code=1)
+
+    targets: list[tuple[str, int, dict | None]] = []
+    for row in rows:
+        if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
+            continue
+        version = _version_info(row)
+        key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
+        targets.append((key, int(row["bgg_id"]), version))
+
+    games_path = cfg.data_dir / "games.json"
+    games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
+
+    need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
+    client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
+
+    fetched: dict[int, dict] = {}
+    blocked = False
+    for start in range(0, len(need), BATCH_SIZE):
+        batch = need[start : start + BATCH_SIZE]
+        try:
+            for thing in client.things_full(batch, refresh=refresh):
+                fetched[thing["bgg_id"]] = thing
+        except BGGAuthError:
+            blocked = True
+            break
+
+    updated = 0
+    for key, bgg_id, version in targets:
+        if bgg_id in fetched:
+            games[key] = {**fetched[bgg_id], "version": version}
+            updated += 1
+
+    games_path.parent.mkdir(parents=True, exist_ok=True)
+    games_path.write_text(json.dumps(games, indent=2, ensure_ascii=False) + "\n")
+
+    typer.echo(
+        f"games.json: {len(games)} entr{'y' if len(games) == 1 else 'ies'} "
+        f"({updated} added/refreshed this run; "
+        f"{len(targets) - updated} already present or waiting)."
+    )
+    if blocked:
+        remaining = [i for i in need if i not in fetched]
+        typer.echo(
+            f"\n{len(remaining)} game(s) are waiting on the BGG API "
+            "(set BGG_API_TOKEN and re-run enrich — everything fetched "
+            "so far is saved)."
+        )
+    return games

src/bggpipe/models.py

@@ -135,6 +135,80 @@ def parse_things(xml_text: str) -> list[ThingDetails]:
     return things
 
 
+def _attr_float(elem: ET.Element | None, attr: str = "value") -> float | None:
+    if elem is None:
+        return None
+    try:
+        return float(elem.get(attr))
+    except (TypeError, ValueError):
+        return None
+
+
+def parse_things_full(xml_text: str) -> list[dict]:
+    """Full game metadata for the enrich stage (games.json). Returns plain
+    dicts — this is artifact data, not pipeline logic."""
+    games = []
+    for item in _root(xml_text).findall("item"):
+
+        def links(link_type: str, item: ET.Element = item) -> list[str]:
+            return [
+                link.get("value", "")
+                for link in item.findall(f"link[@type='{link_type}']")
+            ]
+
+        name = item.find("name[@type='primary']")
+        ratings = item.find("statistics/ratings")
+        best_player_counts = []
+        poll = item.find("poll[@name='suggested_numplayers']")
+        if poll is not None:
+            for results in poll.findall("results"):
+                votes = {
+                    r.get("value"): int(r.get("numvotes") or 0)
+                    for r in results.findall("result")
+                }
+                best = votes.get("Best", 0)
+                if (
+                    best
+                    and best >= votes.get("Recommended", 0)
+                    and best > votes.get("Not Recommended", 0)
+                ):
+                    best_player_counts.append(results.get("numplayers"))
+        games.append(
+            {
+                "bgg_id": int(item.get("id", 0)),
+                "type": item.get("type", "boardgame"),
+                "name": name.get("value", "") if name is not None else "",
+                "year": _attr_int(item.find("yearpublished")),
+                "description": (item.findtext("description") or "").strip(),
+                "image": (item.findtext("image") or "").strip(),
+                "thumbnail": (item.findtext("thumbnail") or "").strip(),
+                "min_players": _attr_int(item.find("minplayers")),
+                "max_players": _attr_int(item.find("maxplayers")),
+                "best_player_counts": best_player_counts,
+                "playtime": _attr_int(item.find("playingtime")),
+                "min_playtime": _attr_int(item.find("minplaytime")),
+                "max_playtime": _attr_int(item.find("maxplaytime")),
+                "min_age": _attr_int(item.find("minage")),
+                "designers": links("boardgamedesigner"),
+                "artists": links("boardgameartist"),
+                "publishers": links("boardgamepublisher"),
+                "categories": links("boardgamecategory"),
+                "mechanics": links("boardgamemechanic"),
+                "rating": _attr_float(ratings.find("average"))
+                if ratings is not None
+                else None,
+                "weight": _attr_float(ratings.find("averageweight"))
+                if ratings is not None
+                else None,
+                "rank": _attr_int(item.find(".//ranks/rank[@name='boardgame']")),
+                "users_owned": _attr_int(ratings.find("owned"))
+                if ratings is not None
+                else None,
+            }
+        )
+    return games
+
+
 def parse_collection(xml_text: str) -> list[CollectionItem]:
     items = []
     for item in _root(xml_text).findall("item"):

tests/test_enrich.py

@@ -0,0 +1,228 @@
+"""Enrich-stage tests: full-metadata parsing plus run_enrich orchestration
+(batched, cache-keyed, idempotent, token-degrading). No network."""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+
+from bggpipe.bgg_client import BGGClient, cache_key
+from bggpipe.config import Config
+from bggpipe.models import parse_things_full
+from bggpipe.resolve import write_matches
+
+FULL_THING_XML = """<items>
+  <item type="boardgame" id="266192">
+    <thumbnail>https://cf.example/thumb.jpg</thumbnail>
+    <image>https://cf.example/full.jpg</image>
+    <name type="primary" sortindex="1" value="Wingspan"/>
+    <name type="alternate" value="Fl&#252;gelschlag"/>
+    <description>A bird-collection engine builder.</description>
+    <yearpublished value="2019"/>
+    <minplayers value="1"/>
+    <maxplayers value="5"/>
+    <poll name="suggested_numplayers">
+      <results numplayers="1">
+        <result value="Best" numvotes="10"/>
+        <result value="Recommended" numvotes="50"/>
+        <result value="Not Recommended" numvotes="20"/>
+      </results>
+      <results numplayers="3">
+        <result value="Best" numvotes="120"/>
+        <result value="Recommended" numvotes="40"/>
+        <result value="Not Recommended" numvotes="2"/>
+      </results>
+      <results numplayers="4">
+        <result value="Best" numvotes="90"/>
+        <result value="Recommended" numvotes="80"/>
+        <result value="Not Recommended" numvotes="5"/>
+      </results>
+    </poll>
+    <playingtime value="70"/>
+    <minplaytime value="40"/>
+    <maxplaytime value="70"/>
+    <minage value="10"/>
+    <link type="boardgamecategory" id="1" value="Animals"/>
+    <link type="boardgamecategory" id="2" value="Card Game"/>
+    <link type="boardgamemechanic" id="3" value="Engine Building"/>
+    <link type="boardgamedesigner" id="4" value="Elizabeth Hargrave"/>
+    <link type="boardgameartist" id="5" value="Natalia Rojas"/>
+    <link type="boardgamepublisher" id="6" value="Stonemaier Games"/>
+    <statistics page="1"><ratings>
+      <average value="8.05"/>
+      <averageweight value="2.45"/>
+      <owned value="123456"/>
+      <ranks><rank type="subtype" id="1" name="boardgame" value="30"/></ranks>
+    </ratings></statistics>
+  </item>
+</items>"""
+
+
+def test_parse_things_full_extracts_everything():
+    (game,) = parse_things_full(FULL_THING_XML)
+    assert game["name"] == "Wingspan"
+    assert game["year"] == 2019
+    assert game["description"] == "A bird-collection engine builder."
+    assert (game["min_players"], game["max_players"]) == (1, 5)
+    # 3 is Best-majority; 4 loses to Recommended+NotRec? no — Best(90) >= Rec(80)
+    # and > NotRec(5), so 4 qualifies too; 1 does not (Best < Recommended)
+    assert game["best_player_counts"] == ["3", "4"]
+    assert game["designers"] == ["Elizabeth Hargrave"]
+    assert game["artists"] == ["Natalia Rojas"]
+    assert game["publishers"] == ["Stonemaier Games"]
+    assert game["categories"] == ["Animals", "Card Game"]
+    assert game["mechanics"] == ["Engine Building"]
+    assert game["rating"] == 8.05
+    assert game["weight"] == 2.45
+    assert game["rank"] == 30
+    assert game["playtime"] == 70
+    assert game["min_age"] == 10
+    assert game["image"].endswith("full.jpg")
+
+
+CATAN_MINIMAL = (
+    '<items><item type="boardgame" id="13">'
+    '<name type="primary" value="CATAN"/><yearpublished value="1995"/>'
+    "</item></items>"
+)
+
+
+def _matches_rows():
+    base = {
+        "year": "",
+        "type": "boardgame",
+        "candidates_json": "[]",
+        "source_photos": "x.jpg",
+        "version_id": "",
+        "version_name": "",
+        "version_status": "version_unknown",
+        "version_candidates_json": "[]",
+    }
+    return [
+        {
+            **base,
+            "title_raw": "Catan",
+            "bgg_id": "13",
+            "bgg_name": "CATAN",
+            "match_status": "auto",
+        },
+        {
+            **base,
+            "title_raw": "Wingspan",
+            "bgg_id": "266192",
+            "bgg_name": "Wingspan",
+            "match_status": "auto",
+            "version_id": "465063",
+            "version_name": "English first edition",
+            "version_status": "version_auto",
+            "version_candidates_json": json.dumps(
+                [
+                    {
+                        "version_id": 465063,
+                        "name": "English first edition",
+                        "year": 2019,
+                        "publishers": ["Stonemaier Games"],
+                        "languages": ["English"],
+                        "score": 5,
+                    }
+                ]
+            ),
+        },
+        {
+            **base,
+            "title_raw": "Junk",
+            "bgg_id": "",
+            "bgg_name": "",
+            "match_status": "rejected",
+        },
+    ]
+
+
+def _no_network(request: httpx.Request) -> httpx.Response:
+    raise AssertionError(f"test hit the network: {request.url}")
+
+
+def _cfg_with_matches(tmp_path):
+    cfg = Config(data_dir=tmp_path / "data")
+    write_matches(cfg.matches_path, _matches_rows())
+    return cfg
+
+
+def _seed_batch_fixture(cache_dir):
+    """The batch cache entry run_enrich will ask for: sorted ids 13,266192."""
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    combined = FULL_THING_XML.replace("<items>", "<items>" + CATAN_MINIMAL[7:-8], 1)
+    key = cache_key("thing", {"id": "13,266192", "stats": "1"})
+    (cache_dir / key).write_text(combined)
+
+
+def test_run_enrich_writes_games_json_with_versions(tmp_path):
+    from bggpipe.enrich import run_enrich
+
+    cfg = _cfg_with_matches(tmp_path)
+    cache = tmp_path / "cache"
+    _seed_batch_fixture(cache)
+    client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network))
+
+    games = run_enrich(cfg, client=client)
+    assert set(games) == {"13", "266192:465063"}
+    wingspan = games["266192:465063"]
+    assert wingspan["designers"] == ["Elizabeth Hargrave"]
+    assert wingspan["version"]["name"] == "English first edition"
+    assert wingspan["version"]["publishers"] == ["Stonemaier Games"]
+    assert games["13"]["version"] is None
+    saved = json.loads((cfg.data_dir / "games.json").read_text())
+    assert saved == games
+
+
+def test_run_enrich_skips_already_enriched(tmp_path):
+    from bggpipe.enrich import run_enrich
+
+    cfg = _cfg_with_matches(tmp_path)
+    cache = tmp_path / "cache"
+    _seed_batch_fixture(cache)
+    client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network))
+    run_enrich(cfg, client=client)
+
+    # empty cache + network-refusing transport: passes only if enrich
+    # doesn't need to fetch anything at all
+    bare = BGGClient(
+        cache_dir=tmp_path / "empty", transport=httpx.MockTransport(_no_network)
+    )
+    games = run_enrich(cfg, client=bare)
+    assert set(games) == {"13", "266192:465063"}
+
+
+def test_refresh_bypasses_cache_read(tmp_path):
+    from bggpipe.enrich import run_enrich
+
+    cfg = _cfg_with_matches(tmp_path)
+    cache = tmp_path / "cache"
+    _seed_batch_fixture(cache)
+    requests = []
+
+    def handler(request):
+        requests.append(str(request.url))
+        return httpx.Response(200, text=FULL_THING_XML.replace("8.05", "7.5"))
+
+    client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(handler))
+    games = run_enrich(cfg, refresh=True, client=client)
+    assert len(requests) == 1  # cache read skipped, live fetch happened
+    assert games["266192:465063"]["rating"] == 7.5
+
+
+def test_enrich_degrades_without_token(tmp_path, capsys):
+    from bggpipe.enrich import run_enrich
+
+    cfg = _cfg_with_matches(tmp_path)
+    client = BGGClient(
+        cache_dir=tmp_path / "empty",
+        transport=httpx.MockTransport(
+            lambda req: httpx.Response(401, text="Unauthorized")
+        ),
+    )
+    games = run_enrich(cfg, client=client)
+    assert games == {}
+    assert "waiting on the BGG API" in capsys.readouterr().out
+    assert json.loads((cfg.data_dir / "games.json").read_text()) == {}