diff --git a/src/bggpipe/bgg_client.py b/src/bggpipe/bgg_client.py
index e148ffe..60257cf 100644
--- a/src/bggpipe/bgg_client.py
+++ b/src/bggpipe/bgg_client.py
@@ -25,6 +25,7 @@ from bggpipe.models import (
parse_collection,
parse_search,
parse_things,
+ parse_things_full,
)
BASE_URL = "https://boardgamegeek.com/xmlapi2"
@@ -83,10 +84,14 @@ class BGGClient:
if wait > 0:
self._sleep(wait)
- def get_xml(self, endpoint: str, params: dict[str, str]) -> str:
- """Fetch one endpoint, serving from and filling the disk cache."""
+ def get_xml(
+ self, endpoint: str, params: dict[str, str], *, refresh: bool = False
+ ) -> str:
+ """Fetch one endpoint, serving from and filling the disk cache.
+ refresh=True skips the cache read (still writes) — for data that
+ drifts over time, like ranks and ratings."""
cache_path = self.cache_dir / cache_key(endpoint, params)
- if cache_path.exists():
+ if cache_path.exists() and not refresh:
return cache_path.read_text()
for attempt in range(MAX_ATTEMPTS):
@@ -142,6 +147,11 @@ class BGGClient:
params["versions"] = "1"
return parse_things(self.get_xml("thing", params))
+ def things_full(self, ids: Iterable[int], *, refresh: bool = False) -> list[dict]:
+ """Full metadata dicts for the enrich stage."""
+ params = {"id": ",".join(str(i) for i in ids), "stats": "1"}
+ return parse_things_full(self.get_xml("thing", params, refresh=refresh))
+
def collection(
self,
username: str,
diff --git a/src/bggpipe/cli.py b/src/bggpipe/cli.py
index e322025..54d23ae 100644
--- a/src/bggpipe/cli.py
+++ b/src/bggpipe/cli.py
@@ -93,4 +93,7 @@ def enrich(
config: ConfigOpt = None,
) -> None:
"""Stage 6: fetch full game + version metadata into games.json."""
- _not_implemented("enrich", 6)
+ from bggpipe.enrich import run_enrich
+
+ cfg = load_config(config)
+ run_enrich(cfg, refresh=refresh)
diff --git a/src/bggpipe/enrich.py b/src/bggpipe/enrich.py
new file mode 100644
index 0000000..449f557
--- /dev/null
+++ b/src/bggpipe/enrich.py
@@ -0,0 +1,107 @@
+"""Stage 6 — enrich: full game + version metadata into games.json.
+
+The seed data for the future web frontend, keyed by bgg_id (or
+"bgg_id:version_id" when a version is settled). Cheap by design:
+- ids are batched (~20 per /thing call) and sorted so batch cache keys
+ stay stable across runs;
+- already-enriched keys are skipped entirely (no request, no cache read)
+ unless --refresh, which bypasses the cache read because ranks and
+ ratings drift over time;
+- the chosen version's details come from matches.csv's stored version
+ candidates — no extra API calls.
+Degrades gracefully without BGG_API_TOKEN: whatever is cached enriches,
+the rest waits for the token.
+"""
+
+from __future__ import annotations
+
+import json
+
+import typer
+
+from bggpipe.bgg_client import BGGAuthError, BGGClient
+from bggpipe.config import Config
+from bggpipe.resolve import read_matches
+
+BATCH_SIZE = 20
+_CONFIDENT_VERSION = ("version_auto", "version_approved")
+
+
+def _version_info(row: dict) -> dict | None:
+ if row["version_status"] not in _CONFIDENT_VERSION or not row["version_id"]:
+ return None
+ version_id = int(row["version_id"])
+ for cand in json.loads(row["version_candidates_json"] or "[]"):
+ if cand.get("version_id") == version_id:
+ return {
+ "version_id": version_id,
+ "name": cand.get("name", ""),
+ "year": cand.get("year"),
+ "publishers": cand.get("publishers") or [],
+ "languages": cand.get("languages") or [],
+ }
+ # candidates were pruned (e.g. manual review) — keep what the row knows
+ return {
+ "version_id": version_id,
+ "name": row["version_name"],
+ "year": None,
+ "publishers": [],
+ "languages": [],
+ }
+
+
+def run_enrich(
+ cfg: Config, *, refresh: bool = False, client: BGGClient | None = None
+) -> dict:
+ rows = read_matches(cfg.matches_path)
+ if not rows:
+ typer.echo(f"{cfg.matches_path} is empty — run `bggpipe resolve` first.")
+ raise typer.Exit(code=1)
+
+ targets: list[tuple[str, int, dict | None]] = []
+ for row in rows:
+ if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
+ continue
+ version = _version_info(row)
+ key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
+ targets.append((key, int(row["bgg_id"]), version))
+
+ games_path = cfg.data_dir / "games.json"
+ games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
+
+ need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
+ client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
+
+ fetched: dict[int, dict] = {}
+ blocked = False
+ for start in range(0, len(need), BATCH_SIZE):
+ batch = need[start : start + BATCH_SIZE]
+ try:
+ for thing in client.things_full(batch, refresh=refresh):
+ fetched[thing["bgg_id"]] = thing
+ except BGGAuthError:
+ blocked = True
+ break
+
+ updated = 0
+ for key, bgg_id, version in targets:
+ if bgg_id in fetched:
+ games[key] = {**fetched[bgg_id], "version": version}
+ updated += 1
+
+ games_path.parent.mkdir(parents=True, exist_ok=True)
+ games_path.write_text(json.dumps(games, indent=2, ensure_ascii=False) + "\n")
+
+ typer.echo(
+ f"games.json: {len(games)} entr{'y' if len(games) == 1 else 'ies'} "
+ f"({updated} added/refreshed this run; "
+ f"{len(targets) - updated} already present or waiting)."
+ )
+ if blocked:
+ remaining = [i for i in need if i not in fetched]
+ typer.echo(
+ f"\n{len(remaining)} game(s) are waiting on the BGG API "
+ "(set BGG_API_TOKEN and re-run enrich — everything fetched "
+ "so far is saved)."
+ )
+ return games
diff --git a/src/bggpipe/models.py b/src/bggpipe/models.py
index bd2f64a..9c96bc0 100644
--- a/src/bggpipe/models.py
+++ b/src/bggpipe/models.py
@@ -135,6 +135,80 @@ def parse_things(xml_text: str) -> list[ThingDetails]:
return things
+def _attr_float(elem: ET.Element | None, attr: str = "value") -> float | None:
+ if elem is None:
+ return None
+ try:
+ return float(elem.get(attr))
+ except (TypeError, ValueError):
+ return None
+
+
+def parse_things_full(xml_text: str) -> list[dict]:
+ """Full game metadata for the enrich stage (games.json). Returns plain
+ dicts — this is artifact data, not pipeline logic."""
+ games = []
+ for item in _root(xml_text).findall("item"):
+
+ def links(link_type: str, item: ET.Element = item) -> list[str]:
+ return [
+ link.get("value", "")
+ for link in item.findall(f"link[@type='{link_type}']")
+ ]
+
+ name = item.find("name[@type='primary']")
+ ratings = item.find("statistics/ratings")
+ best_player_counts = []
+ poll = item.find("poll[@name='suggested_numplayers']")
+ if poll is not None:
+ for results in poll.findall("results"):
+ votes = {
+ r.get("value"): int(r.get("numvotes") or 0)
+ for r in results.findall("result")
+ }
+ best = votes.get("Best", 0)
+ if (
+ best
+ and best >= votes.get("Recommended", 0)
+ and best > votes.get("Not Recommended", 0)
+ ):
+ best_player_counts.append(results.get("numplayers"))
+ games.append(
+ {
+ "bgg_id": int(item.get("id", 0)),
+ "type": item.get("type", "boardgame"),
+ "name": name.get("value", "") if name is not None else "",
+ "year": _attr_int(item.find("yearpublished")),
+ "description": (item.findtext("description") or "").strip(),
+ "image": (item.findtext("image") or "").strip(),
+ "thumbnail": (item.findtext("thumbnail") or "").strip(),
+ "min_players": _attr_int(item.find("minplayers")),
+ "max_players": _attr_int(item.find("maxplayers")),
+ "best_player_counts": best_player_counts,
+ "playtime": _attr_int(item.find("playingtime")),
+ "min_playtime": _attr_int(item.find("minplaytime")),
+ "max_playtime": _attr_int(item.find("maxplaytime")),
+ "min_age": _attr_int(item.find("minage")),
+ "designers": links("boardgamedesigner"),
+ "artists": links("boardgameartist"),
+ "publishers": links("boardgamepublisher"),
+ "categories": links("boardgamecategory"),
+ "mechanics": links("boardgamemechanic"),
+ "rating": _attr_float(ratings.find("average"))
+ if ratings is not None
+ else None,
+ "weight": _attr_float(ratings.find("averageweight"))
+ if ratings is not None
+ else None,
+ "rank": _attr_int(item.find(".//ranks/rank[@name='boardgame']")),
+ "users_owned": _attr_int(ratings.find("owned"))
+ if ratings is not None
+ else None,
+ }
+ )
+ return games
+
+
def parse_collection(xml_text: str) -> list[CollectionItem]:
items = []
for item in _root(xml_text).findall("item"):
diff --git a/tests/test_enrich.py b/tests/test_enrich.py
new file mode 100644
index 0000000..a34c0d4
--- /dev/null
+++ b/tests/test_enrich.py
@@ -0,0 +1,228 @@
+"""Enrich-stage tests: full-metadata parsing plus run_enrich orchestration
+(batched, cache-keyed, idempotent, token-degrading). No network."""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+
+from bggpipe.bgg_client import BGGClient, cache_key
+from bggpipe.config import Config
+from bggpipe.models import parse_things_full
+from bggpipe.resolve import write_matches
+
+FULL_THING_XML = """
+ -
+ https://cf.example/thumb.jpg
+ https://cf.example/full.jpg
+
+
+ A bird-collection engine builder.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+def test_parse_things_full_extracts_everything():
+ (game,) = parse_things_full(FULL_THING_XML)
+ assert game["name"] == "Wingspan"
+ assert game["year"] == 2019
+ assert game["description"] == "A bird-collection engine builder."
+ assert (game["min_players"], game["max_players"]) == (1, 5)
+ # 3 is Best-majority; 4 loses to Recommended+NotRec? no — Best(90) >= Rec(80)
+ # and > NotRec(5), so 4 qualifies too; 1 does not (Best < Recommended)
+ assert game["best_player_counts"] == ["3", "4"]
+ assert game["designers"] == ["Elizabeth Hargrave"]
+ assert game["artists"] == ["Natalia Rojas"]
+ assert game["publishers"] == ["Stonemaier Games"]
+ assert game["categories"] == ["Animals", "Card Game"]
+ assert game["mechanics"] == ["Engine Building"]
+ assert game["rating"] == 8.05
+ assert game["weight"] == 2.45
+ assert game["rank"] == 30
+ assert game["playtime"] == 70
+ assert game["min_age"] == 10
+ assert game["image"].endswith("full.jpg")
+
+
+CATAN_MINIMAL = (
+ '- '
+ ''
+ "
"
+)
+
+
+def _matches_rows():
+ base = {
+ "year": "",
+ "type": "boardgame",
+ "candidates_json": "[]",
+ "source_photos": "x.jpg",
+ "version_id": "",
+ "version_name": "",
+ "version_status": "version_unknown",
+ "version_candidates_json": "[]",
+ }
+ return [
+ {
+ **base,
+ "title_raw": "Catan",
+ "bgg_id": "13",
+ "bgg_name": "CATAN",
+ "match_status": "auto",
+ },
+ {
+ **base,
+ "title_raw": "Wingspan",
+ "bgg_id": "266192",
+ "bgg_name": "Wingspan",
+ "match_status": "auto",
+ "version_id": "465063",
+ "version_name": "English first edition",
+ "version_status": "version_auto",
+ "version_candidates_json": json.dumps(
+ [
+ {
+ "version_id": 465063,
+ "name": "English first edition",
+ "year": 2019,
+ "publishers": ["Stonemaier Games"],
+ "languages": ["English"],
+ "score": 5,
+ }
+ ]
+ ),
+ },
+ {
+ **base,
+ "title_raw": "Junk",
+ "bgg_id": "",
+ "bgg_name": "",
+ "match_status": "rejected",
+ },
+ ]
+
+
+def _no_network(request: httpx.Request) -> httpx.Response:
+ raise AssertionError(f"test hit the network: {request.url}")
+
+
+def _cfg_with_matches(tmp_path):
+ cfg = Config(data_dir=tmp_path / "data")
+ write_matches(cfg.matches_path, _matches_rows())
+ return cfg
+
+
+def _seed_batch_fixture(cache_dir):
+ """The batch cache entry run_enrich will ask for: sorted ids 13,266192."""
+ cache_dir.mkdir(parents=True, exist_ok=True)
+ combined = FULL_THING_XML.replace("", "" + CATAN_MINIMAL[7:-8], 1)
+ key = cache_key("thing", {"id": "13,266192", "stats": "1"})
+ (cache_dir / key).write_text(combined)
+
+
+def test_run_enrich_writes_games_json_with_versions(tmp_path):
+ from bggpipe.enrich import run_enrich
+
+ cfg = _cfg_with_matches(tmp_path)
+ cache = tmp_path / "cache"
+ _seed_batch_fixture(cache)
+ client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network))
+
+ games = run_enrich(cfg, client=client)
+ assert set(games) == {"13", "266192:465063"}
+ wingspan = games["266192:465063"]
+ assert wingspan["designers"] == ["Elizabeth Hargrave"]
+ assert wingspan["version"]["name"] == "English first edition"
+ assert wingspan["version"]["publishers"] == ["Stonemaier Games"]
+ assert games["13"]["version"] is None
+ saved = json.loads((cfg.data_dir / "games.json").read_text())
+ assert saved == games
+
+
+def test_run_enrich_skips_already_enriched(tmp_path):
+ from bggpipe.enrich import run_enrich
+
+ cfg = _cfg_with_matches(tmp_path)
+ cache = tmp_path / "cache"
+ _seed_batch_fixture(cache)
+ client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network))
+ run_enrich(cfg, client=client)
+
+ # empty cache + network-refusing transport: passes only if enrich
+ # doesn't need to fetch anything at all
+ bare = BGGClient(
+ cache_dir=tmp_path / "empty", transport=httpx.MockTransport(_no_network)
+ )
+ games = run_enrich(cfg, client=bare)
+ assert set(games) == {"13", "266192:465063"}
+
+
+def test_refresh_bypasses_cache_read(tmp_path):
+ from bggpipe.enrich import run_enrich
+
+ cfg = _cfg_with_matches(tmp_path)
+ cache = tmp_path / "cache"
+ _seed_batch_fixture(cache)
+ requests = []
+
+ def handler(request):
+ requests.append(str(request.url))
+ return httpx.Response(200, text=FULL_THING_XML.replace("8.05", "7.5"))
+
+ client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(handler))
+ games = run_enrich(cfg, refresh=True, client=client)
+ assert len(requests) == 1 # cache read skipped, live fetch happened
+ assert games["266192:465063"]["rating"] == 7.5
+
+
+def test_enrich_degrades_without_token(tmp_path, capsys):
+ from bggpipe.enrich import run_enrich
+
+ cfg = _cfg_with_matches(tmp_path)
+ client = BGGClient(
+ cache_dir=tmp_path / "empty",
+ transport=httpx.MockTransport(
+ lambda req: httpx.Response(401, text="Unauthorized")
+ ),
+ )
+ games = run_enrich(cfg, client=client)
+ assert games == {}
+ assert "waiting on the BGG API" in capsys.readouterr().out
+ assert json.loads((cfg.data_dir / "games.json").read_text()) == {}