Eric Wagoner
8420a0a1ca
Rows resolving to the same (bgg_id, version_id — or both version-
unknown) are the same physical game read twice unless their extraction
cues conflict (two editions stay separate). The survivor is the read
whose transcription matches the BGG name; losers are marked
match_status=merged with a new merged_into column — no row is ever
deleted, and older matches.csv files without the column still read.
Downstream: diff skips merged rows but folds their photos into the
survivor's to_add provenance; enrich and the review passes ignore them.
The web UI gains a Merges section ("Jokin Ha... merged into Joking
Hazard") with a veto (v key) that restores the row as a distinct
approved match, plus a merged catalog chip and header tally.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
547 lines
18 KiB
Python
547 lines
18 KiB
Python
"""Stage 2 — resolve extracted titles to BGG IDs and versions.
|
||
|
||
Reads data/titles.json, queries BGG search (+ thing stats for tie-breaks,
|
||
+ versions once a game is settled), classifies each title auto/ambiguous/
|
||
unmatched, and appends rows to data/matches.csv. Re-runs skip titles
|
||
already present in matches.csv unless --force.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import csv
|
||
import json
|
||
import os
|
||
import re
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
import typer
|
||
from rapidfuzz import fuzz
|
||
|
||
from bggpipe.bgg_client import BGGAuthError, BGGClient
|
||
from bggpipe.config import Config
|
||
from bggpipe.extract import _cues_conflict
|
||
from bggpipe.models import GameVersion
|
||
from bggpipe.normalize import normalize_title
|
||
|
||
FUZZY_THRESHOLD = 90
|
||
# Tie-break dominance: an exact-named candidate wins outright only if it is
|
||
# clearly the well-known game (spec: obscure duplicates lose to famous ones).
|
||
DOMINANCE_MIN_OWNED = 100
|
||
DOMINANCE_FACTOR = 10
|
||
VERSION_PLAUSIBLE_SCORE = 2
|
||
|
||
MATCH_COLUMNS = [
|
||
"title_raw",
|
||
"bgg_id",
|
||
"bgg_name",
|
||
"year",
|
||
"type",
|
||
"match_status",
|
||
"version_id",
|
||
"version_name",
|
||
"version_status",
|
||
"candidates_json",
|
||
"version_candidates_json",
|
||
"source_photos",
|
||
"merged_into",
|
||
]
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class TitleEntry:
|
||
title_raw: str
|
||
title_normalized: str
|
||
confidence: str = "high"
|
||
publisher_hint: str = ""
|
||
edition_hint: str = ""
|
||
year_hint: int | None = None
|
||
language_hint: str = ""
|
||
art_notes: str = ""
|
||
source_photos: tuple[str, ...] = ()
|
||
|
||
@property
|
||
def has_version_cues(self) -> bool:
|
||
"""Cues strong enough to justify a versions fetch. Language alone
|
||
can never reach the plausibility threshold (it scores 1 of the
|
||
required 2), so fetching on it would waste a rate-limited request —
|
||
it still participates in scoring when other cues exist."""
|
||
return bool(self.publisher_hint or self.edition_hint or self.year_hint)
|
||
|
||
|
||
@dataclass
|
||
class Candidate:
|
||
bgg_id: int
|
||
name: str
|
||
year: int | None
|
||
type: str
|
||
exact: bool
|
||
fuzzy: float
|
||
owned: int | None = None
|
||
rank: int | None = None
|
||
publishers: list[str] = field(default_factory=list)
|
||
|
||
def as_json(self) -> dict:
|
||
return {
|
||
"bgg_id": self.bgg_id,
|
||
"name": self.name,
|
||
"year": self.year,
|
||
"type": self.type,
|
||
"exact": self.exact,
|
||
"fuzzy": round(self.fuzzy, 1),
|
||
"owned": self.owned,
|
||
"rank": self.rank,
|
||
}
|
||
|
||
|
||
@dataclass
|
||
class MatchRow:
|
||
title_raw: str
|
||
source_photos: tuple[str, ...] = ()
|
||
bgg_id: int | None = None
|
||
bgg_name: str = ""
|
||
year: int | None = None
|
||
type: str = ""
|
||
match_status: str = "unmatched"
|
||
version_id: int | None = None
|
||
version_name: str = ""
|
||
version_status: str = ""
|
||
candidates: list[Candidate] = field(default_factory=list)
|
||
version_candidates: list[dict] = field(default_factory=list)
|
||
|
||
def to_csv(self) -> dict[str, str]:
|
||
return {
|
||
"title_raw": self.title_raw,
|
||
"bgg_id": str(self.bgg_id) if self.bgg_id else "",
|
||
"bgg_name": self.bgg_name,
|
||
"year": str(self.year) if self.year else "",
|
||
"type": self.type,
|
||
"match_status": self.match_status,
|
||
"version_id": str(self.version_id) if self.version_id else "",
|
||
"version_name": self.version_name,
|
||
"version_status": self.version_status,
|
||
"candidates_json": json.dumps(
|
||
[c.as_json() for c in self.candidates], ensure_ascii=False
|
||
),
|
||
"version_candidates_json": json.dumps(
|
||
self.version_candidates, ensure_ascii=False
|
||
),
|
||
"source_photos": ";".join(self.source_photos),
|
||
}
|
||
|
||
|
||
def load_titles(path: Path) -> list[TitleEntry]:
|
||
if not path.exists():
|
||
raise FileNotFoundError(
|
||
f"{path} not found — run `bggpipe extract` or hand-write a title list."
|
||
)
|
||
entries = []
|
||
for raw in json.loads(path.read_text()):
|
||
title_raw = raw["title_raw"]
|
||
entries.append(
|
||
TitleEntry(
|
||
title_raw=title_raw,
|
||
title_normalized=raw.get("title_normalized")
|
||
or normalize_title(title_raw),
|
||
confidence=raw.get("confidence", "high"),
|
||
publisher_hint=raw.get("publisher_hint") or "",
|
||
edition_hint=raw.get("edition_hint") or "",
|
||
year_hint=raw.get("year_hint"),
|
||
language_hint=raw.get("language_hint") or "",
|
||
art_notes=raw.get("art_notes") or "",
|
||
source_photos=tuple(raw.get("source_photos") or ()),
|
||
)
|
||
)
|
||
return entries
|
||
|
||
|
||
_SEPARATORS = (" — ", " – ", " - ", ": ", "; ")
|
||
_GAME_WORD = re.compile(r"\s+(?:the\s+|a\s+)?game\b", re.IGNORECASE)
|
||
|
||
|
||
def _truncation_heads(title_raw: str) -> list[str]:
|
||
"""Progressively shorter heads for box titles whose printed subtitle
|
||
defeats search ("CIVILIZATION Game of the Heroic Age - ..."): text
|
||
before the first subtitle separator, before a "(The) Game ..."
|
||
descriptor, then the first two words as a last resort."""
|
||
heads: list[str] = []
|
||
sep_head = next(
|
||
(title_raw.split(sep)[0] for sep in _SEPARATORS if sep in title_raw), None
|
||
)
|
||
if sep_head:
|
||
heads.append(sep_head)
|
||
match = _GAME_WORD.search(title_raw)
|
||
if match and match.start() > 0:
|
||
heads.append(title_raw[: match.start()])
|
||
# last resort: first two words — of the pre-subtitle part, so a title
|
||
# like "Blorvath: Quest of the Zzyzx" never yields "Blorvath: Quest"
|
||
words = (sep_head or title_raw).split()
|
||
if len(words) > 2:
|
||
heads.append(" ".join(words[:2]))
|
||
|
||
seen: set[str] = {normalize_title(title_raw)}
|
||
unique: list[str] = []
|
||
for head in heads:
|
||
norm = normalize_title(head)
|
||
if norm and norm not in seen:
|
||
seen.add(norm)
|
||
unique.append(head)
|
||
return unique[:3]
|
||
|
||
|
||
def _plausible_candidates(
|
||
client: BGGClient, entry: TitleEntry, query: str, head_normalized: str = ""
|
||
) -> list[Candidate]:
|
||
"""Search BGG and keep plausible candidates, one per id: exact-normalized
|
||
or fuzzy>=90 against the FULL title, or — on truncated retries — exact
|
||
(only exact: truncation must stay conservative) against the head."""
|
||
by_id: dict[int, Candidate] = {}
|
||
for result in client.search(query):
|
||
norm = normalize_title(result.name)
|
||
exact = norm == entry.title_normalized
|
||
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
|
||
if not exact and fuzzy < FUZZY_THRESHOLD:
|
||
if not (head_normalized and norm == head_normalized):
|
||
continue
|
||
exact = True # head-exact counts as strong, nothing weaker does
|
||
candidate = Candidate(
|
||
bgg_id=result.bgg_id,
|
||
name=result.name,
|
||
year=result.year,
|
||
type=result.type,
|
||
exact=exact,
|
||
fuzzy=fuzzy,
|
||
)
|
||
prev = by_id.get(result.bgg_id)
|
||
if prev is None or (candidate.exact, candidate.fuzzy) > (
|
||
prev.exact,
|
||
prev.fuzzy,
|
||
):
|
||
by_id[result.bgg_id] = candidate
|
||
return sorted(by_id.values(), key=lambda c: (not c.exact, -c.fuzzy))
|
||
|
||
|
||
def _classify(client: BGGClient, entry: TitleEntry, cands: list[Candidate]) -> MatchRow:
|
||
row = MatchRow(title_raw=entry.title_raw, source_photos=entry.source_photos)
|
||
if not cands:
|
||
row.match_status = "unmatched"
|
||
return row
|
||
|
||
if len(cands) == 1:
|
||
chosen = cands[0]
|
||
else:
|
||
top = cands[:5]
|
||
stats = {
|
||
t.bgg_id: t for t in client.things([c.bgg_id for c in top], stats=True)
|
||
}
|
||
for c in top:
|
||
if c.bgg_id in stats:
|
||
c.owned = stats[c.bgg_id].owned
|
||
c.rank = stats[c.bgg_id].rank
|
||
c.publishers = list(stats[c.bgg_id].publishers)
|
||
row.candidates = top
|
||
chosen = _publisher_pick(entry, top) or _dominant(top)
|
||
if chosen is None:
|
||
row.match_status = "ambiguous"
|
||
return row
|
||
|
||
row.bgg_id = chosen.bgg_id
|
||
row.bgg_name = chosen.name
|
||
row.year = chosen.year
|
||
row.type = chosen.type
|
||
row.match_status = "auto"
|
||
row.candidates = row.candidates or [chosen]
|
||
return row
|
||
|
||
|
||
def _publisher_pick(entry: TitleEntry, top: list[Candidate]) -> Candidate | None:
|
||
"""When a publisher was legible on the box, and exactly one exact-named
|
||
candidate is from that publisher, that's the game."""
|
||
if not entry.publisher_hint:
|
||
return None
|
||
hint = normalize_title(entry.publisher_hint)
|
||
matches = [
|
||
c
|
||
for c in top
|
||
if c.exact
|
||
and any(
|
||
fuzz.partial_ratio(hint, normalize_title(p)) >= 85 for p in c.publishers
|
||
)
|
||
]
|
||
return matches[0] if len(matches) == 1 else None
|
||
|
||
|
||
def _dominant(top: list[Candidate]) -> Candidate | None:
|
||
"""The single clear winner among plausible candidates, if any.
|
||
|
||
Mixed boardgame/expansion candidates are never auto-resolved (the
|
||
spec's most common failure mode); otherwise an exact-named candidate
|
||
wins only when its owned-count dwarfs the runner-up's.
|
||
"""
|
||
if len({c.type for c in top}) > 1:
|
||
return None
|
||
ranked = sorted(top, key=lambda c: c.owned or 0, reverse=True)
|
||
best, second = ranked[0], ranked[1]
|
||
if (
|
||
best.exact
|
||
and (best.owned or 0) >= DOMINANCE_MIN_OWNED
|
||
and (best.owned or 0) >= DOMINANCE_FACTOR * max(second.owned or 0, 1)
|
||
):
|
||
return best
|
||
return None
|
||
|
||
|
||
def _score_version(entry: TitleEntry, version: GameVersion) -> int:
|
||
score = 0
|
||
if entry.publisher_hint:
|
||
hint = normalize_title(entry.publisher_hint)
|
||
if any(
|
||
fuzz.partial_ratio(hint, normalize_title(p)) >= 85
|
||
for p in version.publishers
|
||
):
|
||
score += 2
|
||
if entry.year_hint and version.year == entry.year_hint:
|
||
score += 2
|
||
if entry.language_hint and entry.language_hint.casefold() in {
|
||
lang.casefold() for lang in version.languages
|
||
}:
|
||
score += 1
|
||
if (
|
||
entry.edition_hint
|
||
and version.name
|
||
and fuzz.token_set_ratio(
|
||
normalize_title(entry.edition_hint), normalize_title(version.name)
|
||
)
|
||
>= 80
|
||
):
|
||
score += 2
|
||
return score
|
||
|
||
|
||
def _resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None:
|
||
"""Fill version_* fields on an auto/approved row. Never guess (spec)."""
|
||
if not entry.has_version_cues:
|
||
row.version_status = "version_unknown"
|
||
return
|
||
(thing,) = client.things([row.bgg_id], versions=True)
|
||
scored = sorted(
|
||
((v, _score_version(entry, v)) for v in thing.versions),
|
||
key=lambda pair: -pair[1],
|
||
)
|
||
plausible = [(v, s) for v, s in scored if s >= VERSION_PLAUSIBLE_SCORE]
|
||
if not plausible:
|
||
row.version_status = "version_unknown"
|
||
return
|
||
row.version_candidates = [
|
||
{
|
||
"version_id": v.version_id,
|
||
"name": v.name,
|
||
"year": v.year,
|
||
"publishers": list(v.publishers),
|
||
"languages": list(v.languages),
|
||
"score": s,
|
||
}
|
||
for v, s in plausible[:8]
|
||
]
|
||
if len(plausible) == 1 or plausible[0][1] > plausible[1][1]:
|
||
winner = plausible[0][0]
|
||
row.version_id = winner.version_id
|
||
row.version_name = winner.name
|
||
row.version_status = "version_auto"
|
||
else:
|
||
row.version_status = "version_ambiguous"
|
||
|
||
|
||
def resolve_entry(client: BGGClient, entry: TitleEntry) -> MatchRow:
|
||
cands = _plausible_candidates(client, entry, entry.title_raw)
|
||
if not cands:
|
||
# Long transcribed box titles ("CIVILIZATION Game of the Heroic
|
||
# Age - ...") defeat search: retry with progressively shorter heads
|
||
# instead of loosening the fuzzy threshold.
|
||
for head in _truncation_heads(entry.title_raw):
|
||
cands = _plausible_candidates(client, entry, head, normalize_title(head))
|
||
if cands:
|
||
break
|
||
row = _classify(client, entry, cands)
|
||
if row.match_status == "auto":
|
||
_resolve_version(client, entry, row)
|
||
return row
|
||
|
||
|
||
def _row_key(title_raw: str, source_photos: str) -> tuple[str, str]:
|
||
return (title_raw, source_photos)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class MergeEvent:
|
||
loser_title: str
|
||
survivor_title: str
|
||
bgg_name: str
|
||
bgg_id: str
|
||
|
||
|
||
def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEvent]:
|
||
"""Post-resolve dedupe: rows resolving to the same (bgg_id, version_id —
|
||
or both version-unknown) are the same physical game seen twice (a typo
|
||
read, a partial spine) UNLESS their extraction cues conflict, which
|
||
means two editions. Losers are marked match_status="merged" pointing at
|
||
the survivor via merged_into — no row is ever deleted, and the review
|
||
UI can veto the merge."""
|
||
entry_by_key = {(e.title_raw, ";".join(e.source_photos)): e for e in titles}
|
||
entry_by_title: dict[str, TitleEntry] = {}
|
||
for e in titles:
|
||
entry_by_title.setdefault(e.title_raw, e)
|
||
|
||
def cues(row: dict) -> dict:
|
||
entry = entry_by_key.get(
|
||
(row["title_raw"], row["source_photos"])
|
||
) or entry_by_title.get(row["title_raw"])
|
||
if entry is None:
|
||
return {}
|
||
return {
|
||
"publisher_hint": entry.publisher_hint,
|
||
"edition_hint": entry.edition_hint,
|
||
"language_hint": entry.language_hint,
|
||
"year_hint": entry.year_hint,
|
||
}
|
||
|
||
groups: dict[tuple[str, str], list[dict]] = {}
|
||
for row in rows:
|
||
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
|
||
continue
|
||
confident = (
|
||
row["version_status"] in ("version_auto", "version_approved")
|
||
and row["version_id"]
|
||
)
|
||
key = (row["bgg_id"], row["version_id"] if confident else "")
|
||
groups.setdefault(key, []).append(row)
|
||
|
||
events: list[MergeEvent] = []
|
||
for (bgg_id, _version), group in groups.items():
|
||
if len(group) < 2:
|
||
continue
|
||
# survivor: the row whose transcription best matches the BGG name
|
||
group = sorted(
|
||
group,
|
||
key=lambda r: (
|
||
normalize_title(r["title_raw"]) != normalize_title(r["bgg_name"] or "")
|
||
),
|
||
)
|
||
survivor = group[0]
|
||
for loser in group[1:]:
|
||
if _cues_conflict(cues(survivor), cues(loser)):
|
||
continue # conflicting edition cues: genuinely two copies
|
||
loser["match_status"] = "merged"
|
||
loser["merged_into"] = survivor["title_raw"]
|
||
events.append(
|
||
MergeEvent(
|
||
loser_title=loser["title_raw"],
|
||
survivor_title=survivor["title_raw"],
|
||
bgg_name=survivor["bgg_name"],
|
||
bgg_id=bgg_id,
|
||
)
|
||
)
|
||
return events
|
||
|
||
|
||
def read_matches(path: Path) -> list[dict[str, str]]:
|
||
if not path.exists():
|
||
return []
|
||
with path.open(newline="") as f:
|
||
rows = list(csv.DictReader(f))
|
||
for row in rows: # files written before the merged_into column existed
|
||
row.setdefault("merged_into", "")
|
||
return rows
|
||
|
||
|
||
def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
|
||
"""Atomic full rewrite — review updates rows in place decision by decision."""
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_name(path.name + ".tmp")
|
||
with tmp.open("w", newline="") as f:
|
||
writer = csv.DictWriter(
|
||
f, fieldnames=MATCH_COLUMNS, extrasaction="ignore", restval=""
|
||
)
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
os.replace(tmp, path)
|
||
|
||
|
||
def read_existing_keys(path: Path) -> set[tuple[str, str]]:
|
||
if not path.exists():
|
||
return set()
|
||
with path.open(newline="") as f:
|
||
return {_row_key(r["title_raw"], r["source_photos"]) for r in csv.DictReader(f)}
|
||
|
||
|
||
def append_rows(path: Path, rows: list[MatchRow]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
is_new = not path.exists()
|
||
with path.open("a", newline="") as f:
|
||
writer = csv.DictWriter(f, fieldnames=MATCH_COLUMNS)
|
||
if is_new:
|
||
writer.writeheader()
|
||
for row in rows:
|
||
writer.writerow(row.to_csv())
|
||
|
||
|
||
def run_resolve(
|
||
cfg: Config, *, force: bool = False, client: BGGClient | None = None
|
||
) -> list[MatchRow]:
|
||
entries = load_titles(cfg.titles_path)
|
||
if force and cfg.matches_path.exists():
|
||
cfg.matches_path.unlink()
|
||
existing = read_existing_keys(cfg.matches_path)
|
||
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
||
|
||
new_rows: list[MatchRow] = []
|
||
skipped = 0
|
||
blocked: list[str] = []
|
||
for entry in entries:
|
||
key = _row_key(entry.title_raw, ";".join(entry.source_photos))
|
||
if key in existing:
|
||
skipped += 1
|
||
continue
|
||
try:
|
||
row = resolve_entry(client, entry)
|
||
except BGGAuthError:
|
||
# No API token yet: cached titles still resolve; the rest wait.
|
||
# No row is written, so a future run picks them up untouched.
|
||
blocked.append(entry.title_raw)
|
||
typer.echo(f" {entry.title_raw!r} -> waiting on BGG API token")
|
||
continue
|
||
new_rows.append(row)
|
||
detail = f"{row.bgg_name} ({row.bgg_id})" if row.bgg_id else "-"
|
||
version = f" [{row.version_status}]" if row.version_status else ""
|
||
typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}")
|
||
|
||
append_rows(cfg.matches_path, new_rows)
|
||
|
||
all_rows = read_matches(cfg.matches_path)
|
||
merges = dedupe_matches(all_rows, entries)
|
||
if merges:
|
||
write_matches(cfg.matches_path, all_rows)
|
||
typer.echo("")
|
||
for m in merges:
|
||
typer.echo(
|
||
f" merged {m.loser_title!r} into {m.survivor_title!r} — same "
|
||
f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong"
|
||
)
|
||
|
||
counts: dict[str, int] = {}
|
||
for row in new_rows:
|
||
counts[row.match_status] = counts.get(row.match_status, 0) + 1
|
||
summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items()))
|
||
typer.echo(
|
||
f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); "
|
||
f"skipped {skipped} already in {cfg.matches_path}."
|
||
)
|
||
if blocked:
|
||
typer.echo(
|
||
f"\n{len(blocked)} title(s) are waiting on the BGG API "
|
||
"(register at https://boardgamegeek.com/applications, then set "
|
||
"BGG_API_TOKEN and re-run resolve — everything above is saved): "
|
||
+ ", ".join(blocked)
|
||
)
|
||
return new_rows
|