Audit: 5-reviewer sweep — 19 fixes across every stage, +24 tests

Correctness: review vetoes persist via a dedupe_veto column (resolve
re-runs no longer overturn humans); diff emits second copies whose
confident version matches no owned copy (spec: pairs own only on both
ids) and fetches the live collection with refresh; resolve pairs
titles.json entries to rows by title so a reshoot photo updates
provenance instead of duplicating rows; version lookups survive empty
/thing results; publisher tie-break now honors the mixed
base/expansion veto and refuses multi-candidate picks; empty-normalized
(non-Latin) titles never count as exact.

Upload: LoginError aborts a run instead of logging N bogus failures
(and 3 identical consecutive failures abort as systemic); Cloudflare
interstitials are detected; added-without-version gets its own logged
status that verify understands; same-game updates run one per pass so
the name-targeted row edit can't overwrite a fresh version; absent
diff outputs fail loudly; pagination clicks are paced.

Web review: a lock serializes freshen/decide (threadpool race dropped
decisions); failed saves roll memory back and always alert the browser
(non-JSON 500s included); session warnings reach the page instead of a
StringIO; state-load failures and dead servers show banners instead of
a blank page; duplicate (title, photos) rows are addressable by
ordinal.

Consistency: shared CONFIDENT_VERSION_STATUSES, client_for(),
Config paths for every artifact, one review-port constant, named
matching thresholds, strict collection-id parsing, error-doc responses
never cached, unknown config keys warn, extract reports dropped vision
entries, fixture generators share escaping + marker text.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Eric Wagoner
2026-08-02 14:10:57 -04:00
parent abf7181475
commit 38e20f2c30
26 changed files with 1003 additions and 249 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ Full design lives in `bgg-shelf-pipeline-spec.md` (read it before changing pipel
- `uv sync` — install deps (Python 3.12+, managed by **uv**; use `uv add`, never pip). Playwright needs a one-time `uv run playwright install chromium`.
- `uv run bggpipe <stage>` — run a pipeline stage. Non-secret settings come from `config.toml` (username, dirs, vision model, rate limit); `--config` overrides the path.
- `uv run pytest`105 tests, all offline against fixtures. Tests marked `live` hit the real BGG API (read-only) and are skipped unless you pass `--run-live`.
- `uv run pytest`the suite runs fully offline against fixtures. Tests marked `live` hit the real BGG API (read-only) and are skipped unless you pass `--run-live`.
- `uv run ruff check` / `uv run ruff format` — lint (rules E, F, I, UP, B, SIM) and format.
## Layout
+36
View File
@@ -0,0 +1,36 @@
"""Shared plumbing for the two stub-fixture generators.
Both write into the same cache dirs, so they must agree on XML escaping
(a title containing & or " must not produce malformed XML) and on the
provenance-marker text the upload guard depends on.
"""
from __future__ import annotations
from pathlib import Path
CACHE_MARKER_TEXT = (
"This cache contains hand-written stub XML, not real BGG "
"responses. Data resolved from it must not be uploaded.\n"
)
DATA_MARKER_TEXT = (
"The CSVs in this directory were resolved from hand-written stub "
"fixtures, not real BGG data — version_ids are SYNTHETIC. The "
"upload stage refuses to run while this file exists. Delete it "
"only after re-resolving against real recorded fixtures "
"(BGG_API_TOKEN + scripts/record_fixtures.py + resolve --force).\n"
)
def esc(text: str) -> str:
"""Minimal XML attribute/text escaping for hand-built fixture strings."""
return str(text).replace("&", "&amp;").replace("<", "&lt;").replace('"', "&quot;")
def write_cache_marker(target: Path) -> None:
(target / "STUB_FIXTURES.marker").write_text(CACHE_MARKER_TEXT)
def write_data_marker(data_dir: Path = Path("data")) -> None:
data_dir.mkdir(parents=True, exist_ok=True)
(data_dir / "STUB_DATA.marker").write_text(DATA_MARKER_TEXT)
+10 -21
View File
@@ -17,6 +17,8 @@ from __future__ import annotations
from pathlib import Path
from fixture_common import esc, write_cache_marker, write_data_marker
from bggpipe.bgg_client import cache_key
TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
@@ -295,7 +297,7 @@ VERSIONS: dict[int, list[tuple]] = {
def search_xml(results: list[tuple]) -> str:
items = "".join(
f'<item type="{type_}" id="{bgg_id}">'
f'<name type="{name_type}" value="{_esc(name)}"/>'
f'<name type="{name_type}" value="{esc(name)}"/>'
f'<yearpublished value="{year}"/></item>'
for bgg_id, name, year, type_, name_type in results
)
@@ -306,12 +308,12 @@ def stats_xml(things: list[tuple]) -> str:
items = ""
for bgg_id, name, year, type_, owned, rank, publishers in things:
links = "".join(
f'<link type="boardgamepublisher" id="1" value="{_esc(p)}"/>'
f'<link type="boardgamepublisher" id="1" value="{esc(p)}"/>'
for p in publishers
)
items += (
f'<item type="{type_}" id="{bgg_id}">'
f'<name type="primary" value="{_esc(name)}"/>'
f'<name type="primary" value="{esc(name)}"/>'
f'<yearpublished value="{year}"/>{links}'
f"<statistics><ratings>"
f'<owned value="{owned}"/>'
@@ -326,14 +328,14 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
version_items = ""
for vid, name, year, publishers, languages in versions:
links = "".join(
f'<link type="boardgamepublisher" id="1" value="{_esc(p)}"/>'
f'<link type="boardgamepublisher" id="1" value="{esc(p)}"/>'
for p in publishers
) + "".join(
f'<link type="language" id="1" value="{_esc(lang)}"/>' for lang in languages
f'<link type="language" id="1" value="{esc(lang)}"/>' for lang in languages
)
version_items += (
f'<item type="boardgameversion" id="{vid}">'
f'<name type="primary" value="{_esc(name)}"/>'
f'<name type="primary" value="{esc(name)}"/>'
f'<yearpublished value="{year}"/>{links}</item>'
)
return (
@@ -343,9 +345,6 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
)
def _esc(text: str) -> str:
return text.replace("&", "&amp;").replace("<", "&lt;").replace('"', "&quot;")
def main() -> None:
files: dict[str, str] = {}
@@ -364,18 +363,8 @@ def main() -> None:
(target / name).write_text(xml)
# provenance marker: anything resolved from this cache is stub-derived
# and NOT upload-ready; re-recording real fixtures removes the marker
(target / "STUB_FIXTURES.marker").write_text(
"This cache contains hand-written stub XML, not real BGG "
"responses. Data resolved from it must not be uploaded.\n"
)
DATA_MARKER.parent.mkdir(parents=True, exist_ok=True)
DATA_MARKER.write_text(
"The CSVs in this directory were resolved from hand-written stub "
"fixtures, not real BGG data — version_ids are SYNTHETIC. The "
"upload stage refuses to run while this file exists. Delete it "
"only after re-resolving against real recorded fixtures "
"(BGG_API_TOKEN + scripts/record_fixtures.py + resolve --force).\n"
)
write_cache_marker(target)
write_data_marker(DATA_MARKER.parent)
print(f"Wrote {len(files)} fixture file(s) to {' and '.join(map(str, TARGETS))}")
print(f"Wrote {DATA_MARKER} (committed; upload refuses while it exists)")
+4 -5
View File
@@ -14,6 +14,8 @@ from __future__ import annotations
from pathlib import Path
from fixture_common import esc, write_cache_marker
from bggpipe.bgg_client import cache_key
FIXTURE_CACHE = Path("tests/fixtures/bgg_cache")
@@ -24,7 +26,7 @@ def search_item(bgg_id: int, name: str, year: int | None, type_: str) -> str:
year_xml = f'<yearpublished value="{year}"/>' if year else ""
return (
f'<item type="{type_}" id="{bgg_id}">'
f'<name type="primary" value="{name}"/>{year_xml}</item>'
f'<name type="primary" value="{esc(name)}"/>{year_xml}</item>'
)
@@ -104,10 +106,7 @@ THINGS = {
def main() -> None:
FIXTURE_CACHE.mkdir(parents=True, exist_ok=True)
(FIXTURE_CACHE / "STUB_FIXTURES.marker").write_text(
"This cache contains hand-written stub XML, not real BGG "
"responses. Data resolved from it must not be uploaded.\n"
)
write_cache_marker(FIXTURE_CACHE)
for query, items in SEARCHES.items():
key = cache_key("search", {"query": query, "type": SEARCH_TYPES})
total = items.count("<item ")
+17 -2
View File
@@ -18,7 +18,10 @@ from urllib.parse import urlencode
import httpx
from bggpipe import __version__
from bggpipe.config import Config
from bggpipe.models import (
BGGResponseError,
CollectionItem,
SearchResult,
ThingDetails,
@@ -65,7 +68,7 @@ class BGGClient:
self._monotonic = monotonic
self._rng = rng or random.Random()
self._last_request: float | None = None
headers = {"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"}
headers = {"User-Agent": f"bggpipe/{__version__} (shelf-collection pipeline)"}
# BGG requires registered applications since 2025: the token from
# https://boardgamegeek.com/applications must accompany every request.
# Env var only — never config, disk, or logs.
@@ -117,6 +120,12 @@ class BGGClient:
continue
response.raise_for_status()
if "<errors" in response.text[:120]:
# BGG serves some errors as HTTP 200 <errors> XML (bad
# username etc.) — caching one would poison every re-run
raise BGGResponseError(
f"BGG error document for /{endpoint}: {response.text[:200]}"
)
self.cache_dir.mkdir(parents=True, exist_ok=True)
cache_path.write_text(response.text)
return response.text
@@ -137,6 +146,7 @@ class BGGClient:
def things(
self,
ids: Iterable[int],
*,
stats: bool = False,
versions: bool = False,
) -> list[ThingDetails]:
@@ -155,9 +165,9 @@ class BGGClient:
def collection(
self,
username: str,
*,
subtype: str | None = None,
version: bool = True,
*,
refresh: bool = False,
) -> list[CollectionItem]:
params = {"username": username, "own": "1"}
@@ -179,3 +189,8 @@ class BGGClient:
)
seen = {item.coll_id for item in base}
return base + [e for e in expansions if e.coll_id not in seen]
def client_for(cfg: Config) -> BGGClient:
"""The standard injection fallback: every stage's `client or client_for(cfg)`."""
return BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
+8 -3
View File
@@ -7,7 +7,7 @@ from typing import Annotated
import typer
from bggpipe.config import load_config
from bggpipe.config import DEFAULT_REVIEW_PORT, load_config
app = typer.Typer(
help="Shelf-to-BGG collection pipeline.",
@@ -56,7 +56,9 @@ def review(
web: Annotated[
bool, typer.Option("--web", help="Serve the review UI on localhost")
] = False,
port: Annotated[int, typer.Option("--port", help="Port for --web")] = 8377,
port: Annotated[
int, typer.Option("--port", help="Port for --web")
] = DEFAULT_REVIEW_PORT,
dev: Annotated[
bool, typer.Option("--dev", help="With --web: restart on source changes")
] = False,
@@ -122,7 +124,10 @@ def upload(
@app.command()
def enrich(
refresh: Annotated[bool, typer.Option("--refresh")] = False,
refresh: Annotated[
bool,
typer.Option("--refresh", help="Re-fetch metadata (ranks/ratings drift)"),
] = False,
config: ConfigOpt = None,
) -> None:
"""Stage 6: fetch full game + version metadata into games.json."""
+41
View File
@@ -10,10 +10,12 @@ from __future__ import annotations
import os
import tomllib
import warnings
from dataclasses import dataclass, replace
from pathlib import Path
DEFAULT_CONFIG_PATH = Path("config.toml")
DEFAULT_REVIEW_PORT = 8377
@dataclass(frozen=True)
@@ -40,6 +42,38 @@ class Config:
def matches_path(self) -> Path:
return self.data_dir / "matches.csv"
@property
def extract_raw_dir(self) -> Path:
return self.data_dir / "extract_raw"
@property
def to_add_path(self) -> Path:
return self.data_dir / "to_add.csv"
@property
def to_update_path(self) -> Path:
return self.data_dir / "to_update.csv"
@property
def upload_log_path(self) -> Path:
return self.data_dir / "upload_log.csv"
@property
def games_path(self) -> Path:
return self.data_dir / "games.json"
@property
def dismissed_path(self) -> Path:
return self.data_dir / "unidentified_dismissed.json"
@property
def stub_marker_paths(self) -> tuple[Path, Path]:
# gitignored (travels with the stub XML) + committed (guards clones)
return (
self.cache_dir / "STUB_FIXTURES.marker",
self.data_dir / "STUB_DATA.marker",
)
def load_config(path: Path | None = None) -> Config:
cfg = Config()
@@ -53,6 +87,13 @@ def load_config(path: Path | None = None) -> Config:
"rate_limit_seconds": float,
}
updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw}
if unknown := sorted(raw.keys() - known.keys()):
# a typo'd key silently falling back to defaults is a debugging
# trap ("No photos found in photos/") — say so up front
warnings.warn(
f"{p}: ignoring unknown key(s): {', '.join(unknown)}",
stacklevel=2,
)
cfg = replace(cfg, **updates)
if username := os.environ.get("BGG_USERNAME"):
cfg = replace(cfg, bgg_username=username)
+47 -35
View File
@@ -8,11 +8,12 @@ Two collection sources:
logged-in-user exemption.
Outputs both artifacts:
- to_add.csv — recognized games not in the collection (new entries);
- to_add.csv — recognized games not in the collection, including
additional copies whose confident version matches no owned copy;
- to_update.csv — owned, VERSION-LESS entries where matching produced a
confident version (version_auto/version_approved). Strictly additive:
entries that already carry a version are never touched — a photo/version
disagreement is reported in the summary instead.
entries that already carry a version are never touched — a further copy
with a different version becomes a to_add row instead.
"""
from __future__ import annotations
@@ -24,9 +25,13 @@ from pathlib import Path
import typer
from bggpipe.bgg_client import BGGClient
from bggpipe.bgg_client import BGGClient, client_for
from bggpipe.config import Config
from bggpipe.models import CollectionItem, parse_collection
from bggpipe.models import (
CONFIDENT_VERSION_STATUSES,
CollectionItem,
parse_collection,
)
from bggpipe.resolve import read_matches
TO_ADD_COLUMNS = [
@@ -42,7 +47,6 @@ TO_ADD_COLUMNS = [
TO_UPDATE_COLUMNS = ["collid", "bgg_id", "bgg_name", "version_id", "version_name"]
SNAPSHOT_FILES = ("collection_snapshot_base.xml", "collection_snapshot_expansions.xml")
_CONFIDENT_VERSION = ("version_auto", "version_approved")
@dataclass
@@ -50,7 +54,7 @@ class DiffResult:
to_add: list[dict] = field(default_factory=list)
to_update: list[dict] = field(default_factory=list)
already_owned: list[str] = field(default_factory=list) # title_raw
disagreements: list[str] = field(default_factory=list)
second_copies: list[str] = field(default_factory=list) # notes for adds
unseen: list[CollectionItem] = field(default_factory=list)
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
rejected: int = 0
@@ -113,24 +117,27 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
if copies:
seen_object_ids.add(bgg_id)
confident = row["version_status"] in _CONFIDENT_VERSION and row["version_id"]
confident = (
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
)
version_id = int(row["version_id"]) if confident else None
if not copies:
def add_row(row: dict = row, confident: bool = confident) -> dict:
photos = {p for p in row["source_photos"].split(";") if p}
photos |= merged_photos.get(row["title_raw"], set())
result.to_add.append(
{
"bgg_id": row["bgg_id"],
"bgg_name": row["bgg_name"],
"year": row["year"],
"type": row["type"],
"version_id": row["version_id"] if confident else "",
"version_name": row["version_name"] if confident else "",
"title_raw": row["title_raw"],
"source_photos": ";".join(sorted(photos)),
}
)
return {
"bgg_id": row["bgg_id"],
"bgg_name": row["bgg_name"],
"year": row["year"],
"type": row["type"],
"version_id": row["version_id"] if confident else "",
"version_name": row["version_name"] if confident else "",
"title_raw": row["title_raw"],
"source_photos": ";".join(sorted(photos)),
}
if not copies:
result.to_add.append(add_row())
continue
if not confident:
@@ -161,12 +168,15 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
}
)
else:
# every copy already carries a (different) version — never touch it
result.already_owned.append(row["title_raw"])
result.disagreements.append(
f"{row['title_raw']}: photo suggests version "
f"{row['version_name']!r} ({row['version_id']}) but the "
"collection entry already has a version set — not changing it"
# no remaining copy can take this version: every copy either
# already carries a different version or was consumed by another
# match — per spec this is an additional physical copy to ADD
# (existing entries are never touched)
result.to_add.append(add_row())
result.second_copies.append(
f"{row['title_raw']}: adding as a NEW copy with version "
f"{row['version_name']!r} ({row['version_id']}) — every "
"existing entry of this game keeps its current version"
)
result.unseen = [
@@ -177,10 +187,12 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="") as f:
tmp = path.with_name(path.name + ".tmp")
with tmp.open("w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore")
writer.writeheader()
writer.writerows(rows)
os.replace(tmp, path) # atomic: a killed diff never leaves a torn queue
def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
@@ -191,8 +203,8 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
if os.environ.get("BGG_API_TOKEN") and cfg.bgg_username:
typer.echo("Fetching live collection from BGG…")
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
collection = client.collection_full(cfg.bgg_username)
client = client or client_for(cfg)
collection = client.collection_full(cfg.bgg_username, refresh=True)
else:
typer.echo(
"No BGG_API_TOKEN — using collection snapshot files in "
@@ -202,8 +214,8 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
result = compute_diff(rows, collection)
_write_csv(cfg.data_dir / "to_add.csv", TO_ADD_COLUMNS, result.to_add)
_write_csv(cfg.data_dir / "to_update.csv", TO_UPDATE_COLUMNS, result.to_update)
_write_csv(cfg.to_add_path, TO_ADD_COLUMNS, result.to_add)
_write_csv(cfg.to_update_path, TO_UPDATE_COLUMNS, result.to_update)
merged_note = f" · {result.merged} merged duplicate(s)" if result.merged else ""
typer.echo(
@@ -212,9 +224,9 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
f"update(s) · {len(result.pending)} pending review · "
f"{result.rejected} rejected{merged_note}"
)
if result.disagreements:
typer.echo("\nVersion disagreements (left untouched):")
for line in result.disagreements:
if result.second_copies:
typer.echo("\nSecond copies (verify these on the dry run before upload):")
for line in result.second_copies:
typer.echo(f" - {line}")
if result.pending:
typer.echo("\nStill pending review: " + ", ".join(result.pending))
+1 -1
View File
@@ -66,7 +66,7 @@ def run_enrich(
key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
targets.append((key, int(row["bgg_id"]), version))
games_path = cfg.data_dir / "games.json"
games_path = cfg.games_path
games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
+17 -7
View File
@@ -103,7 +103,7 @@ def prepare_image(path: Path) -> tuple[str, str]:
return base64.standard_b64encode(buffer.getvalue()).decode(), "image/jpeg"
def parse_vision_response(text: str) -> tuple[list[dict], list[dict]]:
def parse_vision_response(text: str) -> tuple[list[dict], list[dict], int]:
"""Parse the model's JSON defensively: strip code fences, locate the
payload amid any prose. Returns (title entries, unidentified sightings).
A bare JSON array (the pre-unidentified response shape) still parses —
@@ -133,7 +133,8 @@ def parse_vision_response(text: str) -> tuple[list[dict], list[dict]]:
if isinstance(u, dict)
and (u.get("location") or u.get("partial_text") or u.get("art_notes"))
]
return titles, unidentified
dropped = len(titles_raw) - len(titles)
return titles, unidentified, dropped
def default_vision(model: str) -> VisionFn:
@@ -172,7 +173,9 @@ def extract_photo(photo: Path, vision: VisionFn) -> dict:
"""One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache
file format)."""
image_b64, media_type = prepare_image(photo)
raw_entries, raw_unidentified = parse_vision_response(vision(image_b64, media_type))
raw_entries, raw_unidentified, dropped = parse_vision_response(
vision(image_b64, media_type)
)
unidentified = [
{
"location": str(u.get("location") or "").strip(),
@@ -198,10 +201,10 @@ def extract_photo(photo: Path, vision: VisionFn) -> dict:
"source_photos": [photo.name],
}
)
return {"titles": entries, "unidentified": unidentified}
return {"titles": entries, "unidentified": unidentified, "dropped": dropped}
def _cues_conflict(a: dict, b: dict) -> bool:
def cues_conflict(a: dict, b: dict) -> bool:
"""Two sightings conflict if any edition cue is set on both and differs —
that means visibly different boxes, i.e. separate editions (spec)."""
for key in ("publisher_hint", "edition_hint", "language_hint"):
@@ -238,7 +241,7 @@ def dedupe_entries(entries: list[dict]) -> list[dict]:
for existing in result:
if existing["title_normalized"] == entry[
"title_normalized"
] and not _cues_conflict(existing, entry):
] and not cues_conflict(existing, entry):
existing.update(_merge(existing, entry))
break
else:
@@ -298,7 +301,7 @@ def run_extract(
)
raise typer.Exit(code=1)
raw_dir = cfg.data_dir / "extract_raw"
raw_dir = cfg.extract_raw_dir
raw_dir.mkdir(parents=True, exist_ok=True)
vision = vision or default_vision(cfg.model)
@@ -314,6 +317,13 @@ def run_extract(
if result["unidentified"]
else ""
)
if result.get("dropped"):
# the prompt forbids silently omitting a box; so do we
note += (
f" — DROPPED {result['dropped']} malformed entr"
f"{'y' if result['dropped'] == 1 else 'ies'}; inspect "
f"{raw_path}"
)
typer.echo(f" {photo.name}: {len(result['titles'])} title(s){note}")
deduped, unidentified = rebuild_artifacts(
+20 -2
View File
@@ -12,6 +12,12 @@ class BGGResponseError(Exception):
"""The API returned a well-formed error document (e.g. bad username)."""
# The one status predicate the whole pipeline shares: a version is trusted
# for diff/upload/enrich only when matching produced it confidently or a
# human approved it.
CONFIDENT_VERSION_STATUSES = ("version_auto", "version_approved")
@dataclass(frozen=True)
class SearchResult:
bgg_id: int
@@ -209,6 +215,16 @@ def parse_things_full(xml_text: str) -> list[dict]:
return games
def _required_attr(item, name: str) -> str:
value = item.get(name)
if not value:
raise BGGResponseError(
f"collection item missing {name!r} — truncated or unexpected "
"response; refusing to feed it to the diff"
)
return value
def parse_collection(xml_text: str) -> list[CollectionItem]:
items = []
for item in _root(xml_text).findall("item"):
@@ -217,8 +233,10 @@ def parse_collection(xml_text: str) -> list[CollectionItem]:
version_item = item.find("version/item")
items.append(
CollectionItem(
object_id=int(item.get("objectid", 0)),
coll_id=int(item.get("collid", 0)),
# strict: a missing id coerced to 0 would collide in the
# collid dedupe and silently drop owned games from the diff
object_id=int(_required_attr(item, "objectid")),
coll_id=int(_required_attr(item, "collid")),
name=item.findtext("name", default=""),
subtype=item.get("subtype", "boardgame"),
own=status is not None and status.get("own") == "1",
+70 -47
View File
@@ -12,16 +12,17 @@ import csv
import json
import os
import re
from collections import Counter
from dataclasses import dataclass, field
from pathlib import Path
import typer
from rapidfuzz import fuzz
from bggpipe.bgg_client import BGGAuthError, BGGClient
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
from bggpipe.config import Config
from bggpipe.extract import _cues_conflict
from bggpipe.models import GameVersion
from bggpipe.extract import cues_conflict
from bggpipe.models import CONFIDENT_VERSION_STATUSES, GameVersion
from bggpipe.normalize import normalize_title
FUZZY_THRESHOLD = 90
@@ -30,6 +31,10 @@ FUZZY_THRESHOLD = 90
DOMINANCE_MIN_OWNED = 100
DOMINANCE_FACTOR = 10
VERSION_PLAUSIBLE_SCORE = 2
# "Is this the publisher on the box?" — one answer, asked in two places
# (candidate tie-break and version scoring): the sites must move together.
PUBLISHER_MATCH_THRESHOLD = 85
EDITION_NAME_THRESHOLD = 80
MATCH_COLUMNS = [
"title_raw",
@@ -45,6 +50,7 @@ MATCH_COLUMNS = [
"version_candidates_json",
"source_photos",
"merged_into",
"dedupe_veto",
]
@@ -127,6 +133,8 @@ class MatchRow:
self.version_candidates, ensure_ascii=False
),
"source_photos": ";".join(self.source_photos),
"merged_into": "",
"dedupe_veto": "",
}
@@ -198,6 +206,10 @@ def _plausible_candidates(
by_id: dict[int, Candidate] = {}
for result in client.search(query):
norm = normalize_title(result.name)
# a fully non-Latin title normalizes to "" — empty-vs-empty is not a
# match (and token_sort_ratio("", "") is 100, so guard fuzzy too)
if not entry.title_normalized:
continue
exact = norm == entry.title_normalized
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
if not exact and fuzzy < FUZZY_THRESHOLD:
@@ -259,13 +271,19 @@ def _publisher_pick(entry: TitleEntry, top: list[Candidate]) -> Candidate | None
candidate is from that publisher, that's the game."""
if not entry.publisher_hint:
return None
if len({c.type for c in top}) > 1:
# mixed base-game/expansion candidates are never auto-resolved —
# the same veto _dominant applies (spec's top failure mode); an
# alternate name can make an expansion "exact" too
return None
hint = normalize_title(entry.publisher_hint)
matches = [
c
for c in top
if c.exact
and any(
fuzz.partial_ratio(hint, normalize_title(p)) >= 85 for p in c.publishers
fuzz.partial_ratio(hint, normalize_title(p)) >= PUBLISHER_MATCH_THRESHOLD
for p in c.publishers
)
]
return matches[0] if len(matches) == 1 else None
@@ -296,7 +314,7 @@ def _score_version(entry: TitleEntry, version: GameVersion) -> int:
if entry.publisher_hint:
hint = normalize_title(entry.publisher_hint)
if any(
fuzz.partial_ratio(hint, normalize_title(p)) >= 85
fuzz.partial_ratio(hint, normalize_title(p)) >= PUBLISHER_MATCH_THRESHOLD
for p in version.publishers
):
score += 2
@@ -312,18 +330,24 @@ def _score_version(entry: TitleEntry, version: GameVersion) -> int:
and fuzz.token_set_ratio(
normalize_title(entry.edition_hint), normalize_title(version.name)
)
>= 80
>= EDITION_NAME_THRESHOLD
):
score += 2
return score
def _resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None:
"""Fill version_* fields on an auto/approved row. Never guess (spec)."""
def resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None:
"""Fill version_* fields on an auto/approved row. Never guess (spec).
Public: review's manual-id path calls this too."""
if not entry.has_version_cues:
row.version_status = "version_unknown"
return
(thing,) = client.things([row.bgg_id], versions=True)
things = client.things([row.bgg_id], versions=True)
if not things:
# a manually-typed id BGG doesn't know: nothing to offer
row.version_status = "version_unknown"
return
thing = things[0]
scored = sorted(
((v, _score_version(entry, v)) for v in thing.versions),
key=lambda pair: -pair[1],
@@ -364,14 +388,10 @@ def resolve_entry(client: BGGClient, entry: TitleEntry) -> MatchRow:
break
row = _classify(client, entry, cands)
if row.match_status == "auto":
_resolve_version(client, entry, row)
resolve_version(client, entry, row)
return row
def _row_key(title_raw: str, source_photos: str) -> tuple[str, str]:
return (title_raw, source_photos)
@dataclass(frozen=True)
class MergeEvent:
loser_title: str
@@ -409,9 +429,13 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
for row in rows:
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
continue
if row.get("dedupe_veto"):
# a human already ruled "this is a genuinely separate copy" —
# re-running resolve must never overturn that (spec: re-runs
# lose no work, least of all review decisions)
continue
confident = (
row["version_status"] in ("version_auto", "version_approved")
and row["version_id"]
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
)
key = (row["bgg_id"], row["version_id"] if confident else "")
groups.setdefault(key, []).append(row)
@@ -429,7 +453,7 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
)
survivor = group[0]
for loser in group[1:]:
if _cues_conflict(cues(survivor), cues(loser)):
if cues_conflict(cues(survivor), cues(loser)):
continue # conflicting edition cues: genuinely two copies
loser["match_status"] = "merged"
loser["merged_into"] = survivor["title_raw"]
@@ -449,8 +473,9 @@ def read_matches(path: Path) -> list[dict[str, str]]:
return []
with path.open(newline="") as f:
rows = list(csv.DictReader(f))
for row in rows: # files written before the merged_into column existed
for row in rows: # files written before these columns existed
row.setdefault("merged_into", "")
row.setdefault("dedupe_veto", "")
return rows
@@ -467,39 +492,40 @@ def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
os.replace(tmp, path)
def read_existing_keys(path: Path) -> set[tuple[str, str]]:
if not path.exists():
return set()
with path.open(newline="") as f:
return {_row_key(r["title_raw"], r["source_photos"]) for r in csv.DictReader(f)}
def append_rows(path: Path, rows: list[MatchRow]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
is_new = not path.exists()
with path.open("a", newline="") as f:
writer = csv.DictWriter(f, fieldnames=MATCH_COLUMNS)
if is_new:
writer.writeheader()
for row in rows:
writer.writerow(row.to_csv())
def run_resolve(
cfg: Config, *, force: bool = False, client: BGGClient | None = None
) -> list[MatchRow]:
entries = load_titles(cfg.titles_path)
if force and cfg.matches_path.exists():
cfg.matches_path.unlink()
existing = read_existing_keys(cfg.matches_path)
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
existing_rows = read_matches(cfg.matches_path)
client = client or client_for(cfg)
# Pair entries with existing rows BY TITLE, positionally, not by exact
# (title, photos) key: extract unions a new photo of an already-resolved
# game into its entry, and that must update the row's provenance — not
# re-resolve the game as a duplicate row. Same-title entries only stay
# separate when their cues conflict (two editions), and those pair up
# in stable file order on both sides.
rows_by_title: dict[str, list[dict]] = {}
for row in existing_rows:
rows_by_title.setdefault(row["title_raw"], []).append(row)
seen_per_title: dict[str, int] = {}
new_rows: list[MatchRow] = []
skipped = 0
photos_updated = False
blocked: list[str] = []
for entry in entries:
key = _row_key(entry.title_raw, ";".join(entry.source_photos))
if key in existing:
ix = seen_per_title.get(entry.title_raw, 0)
seen_per_title[entry.title_raw] = ix + 1
paired = rows_by_title.get(entry.title_raw, [])
if ix < len(paired):
row_dict = paired[ix]
photos = ";".join(entry.source_photos)
if row_dict["source_photos"] != photos:
row_dict["source_photos"] = photos
photos_updated = True
skipped += 1
continue
try:
@@ -515,12 +541,11 @@ def run_resolve(
version = f" [{row.version_status}]" if row.version_status else ""
typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}")
append_rows(cfg.matches_path, new_rows)
all_rows = read_matches(cfg.matches_path)
all_rows = existing_rows + [row.to_csv() for row in new_rows]
merges = dedupe_matches(all_rows, entries)
if new_rows or photos_updated or merges:
write_matches(cfg.matches_path, all_rows) # atomic full rewrite
if merges:
write_matches(cfg.matches_path, all_rows)
typer.echo("")
for m in merges:
typer.echo(
@@ -528,9 +553,7 @@ def run_resolve(
f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong"
)
counts: dict[str, int] = {}
for row in new_rows:
counts[row.match_status] = counts.get(row.match_status, 0) + 1
counts = Counter(row.match_status for row in new_rows)
summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items()))
typer.echo(
f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); "
+75 -30
View File
@@ -1,9 +1,11 @@
"""Stage 3 — human review of ambiguous/unmatched matches, then versions.
"""Stage 3 — the review decision engine, worn by two faces.
A plain rich prompt loop (deliberately dependency-light, per spec). Every
decision rewrites matches.csv atomically, so quitting mid-review q,
Ctrl-C, or end of input loses nothing; the next run resumes with
whatever is still ambiguous/unmatched. The version pass is optional and
ReviewSession owns all decision logic and every matches.csv write: the
decision API (decide_*/veto_merge/pending_rows/version_rows) serves both
this module's rich TUI prompt loop and webreview's FastAPI endpoints, and
reload_if_changed() lets either face follow external rewrites (a resolve
run in another terminal). Every decision rewrites matches.csv atomically,
so quitting mid-review loses nothing. The version pass is optional and
skippable: version review must never block getting games uploaded.
"""
@@ -14,18 +16,20 @@ from collections.abc import Callable
from pathlib import Path
import httpx
import typer
from rich.console import Console
from rich.panel import Panel
from rich.table import Table
from bggpipe.bgg_client import BGGAuthError, BGGClient, BGGQueueTimeout
from bggpipe.bgg_client import BGGAuthError, BGGClient, BGGQueueTimeout, client_for
from bggpipe.config import Config
from bggpipe.models import BGGResponseError
from bggpipe.resolve import (
MatchRow,
_resolve_version,
TitleEntry,
load_titles,
read_matches,
resolve_version,
write_matches,
)
@@ -57,8 +61,9 @@ class ReviewSession:
self.cfg = cfg
self.console = console or Console()
self.input_fn = input_fn or (lambda prompt: self.console.input(prompt))
self.client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
self.client = client or client_for(cfg)
self.decisions = 0
self.warnings: list[str] = []
self._load()
# -- plumbing -------------------------------------------------------
@@ -99,8 +104,34 @@ class ReviewSession:
raise _Quit
return answer
def _save(self) -> None:
write_matches(self.cfg.matches_path, self.rows)
def _warn(self, message: str) -> None:
"""Degradations must be visible in BOTH faces: the console for the
TUI, and self.warnings for the web UI (whose console is a StringIO)."""
self.warnings.append(message)
self.console.print(f"[yellow]{message}[/yellow]")
def _save(self, row: dict | None = None) -> None:
"""Atomic write of the in-memory rows. If another process rewrote the
file since we loaded (resolve in a second terminal), reload first and
re-apply `row` the one decision being saved by identity, so
neither side's work is silently lost."""
if self.reload_if_changed() and row is not None:
key = (row["title_raw"], row["source_photos"])
for i, fresh in enumerate(self.rows):
if (fresh["title_raw"], fresh["source_photos"]) == key:
self.rows[i] = row
break
else:
self._warn(
f"{row['title_raw']!r} disappeared from matches.csv while "
"you decided — decision NOT saved"
)
return
try:
write_matches(self.cfg.matches_path, self.rows)
except OSError:
self._load() # memory must never claim what disk doesn't hold
raise
self._loaded_mtimes = self._data_mtimes() # own writes aren't "changes"
self.decisions += 1
@@ -111,7 +142,7 @@ class ReviewSession:
row["year"] = str(candidate.get("year") or "")
row["type"] = candidate.get("type") or "boardgame"
self._fill_version(row)
self._save()
self._save(row)
def _fill_version(self, row: dict) -> None:
"""Try version resolution for a just-approved row. Degrades gracefully:
@@ -122,9 +153,9 @@ class ReviewSession:
return
shim = MatchRow(title_raw=row["title_raw"], bgg_id=int(row["bgg_id"]))
try:
_resolve_version(self.client, entry, shim)
resolve_version(self.client, entry, shim)
except _BGG_ERRORS as err:
self.console.print(f"[yellow]version lookup unavailable: {err}[/yellow]")
self._warn(f"version lookup unavailable ({err}) — recorded version_unknown")
row["version_status"] = "version_unknown"
return
row["version_status"] = shim.version_status
@@ -155,9 +186,21 @@ class ReviewSession:
row as a distinct, human-confirmed match."""
row["match_status"] = "approved"
row["merged_into"] = ""
self._save()
row["dedupe_veto"] = "1" # persists: resolve re-runs must not re-merge
self._save(row)
def cues_for(self, title_raw: str):
def cues_for(
self, title_raw: str, source_photos: str | None = None
) -> TitleEntry | None:
"""The extraction entry behind a row. Same-title entries (two
editions of one game) are told apart by their photo set when given."""
if source_photos is not None:
for entry in self.titles:
if (
entry.title_raw == title_raw
and ";".join(entry.source_photos) == source_photos
):
return entry
return self._titles.get(title_raw)
def decide_pick(self, row: dict, candidate: dict) -> None:
@@ -168,7 +211,7 @@ class ReviewSession:
def decide_reject(self, row: dict) -> None:
row["match_status"] = "rejected"
self._save()
self._save(row)
def decide_version(self, row: dict, version_id: int | None) -> None:
"""Pick a version from the row's stored candidates, or None -> unknown."""
@@ -186,7 +229,7 @@ class ReviewSession:
row["version_status"] = "version_approved"
row["version_id"] = str(version_id)
row["version_name"] = chosen.get("name") or ""
self._save()
self._save(row)
# -- displays -------------------------------------------------------
@@ -247,13 +290,20 @@ class ReviewSession:
def _manual_id(self, row: dict, bgg_id: int) -> None:
candidate: dict = {"bgg_id": bgg_id}
try:
(thing,) = self.client.things([bgg_id])
candidate.update(name=thing.name, year=thing.year, type=thing.type)
things = self.client.things([bgg_id])
except _BGG_ERRORS as err:
self.console.print(
f"[yellow]couldn't look up id {bgg_id} ({err}) — "
"recording the id with no name[/yellow]"
self._warn(
f"couldn't look up id {bgg_id} ({err}) — recording the id with no name"
)
else:
if things:
thing = things[0]
candidate.update(name=thing.name, year=thing.year, type=thing.type)
else:
self._warn(
f"BGG knows no game with id {bgg_id} — recording it "
"with no name (typo?)"
)
self._apply_choice(row, candidate)
def _research(self, query: str) -> list[dict]:
@@ -302,15 +352,10 @@ class ReviewSession:
# -- entry point ----------------------------------------------------
def run(self) -> None:
pending = [
r for r in self.rows if r["match_status"] in ("ambiguous", "unmatched")
]
try:
for row in pending:
for row in self.pending_rows():
self._review_match_row(row)
versions = [
r for r in self.rows if r["version_status"] == "version_ambiguous"
]
versions = self.version_rows()
if versions:
answer = self._ask(
f"{len(versions)} game(s) have ambiguous editions. "
@@ -343,6 +388,6 @@ def run_review(
session.console.print(
f"{cfg.matches_path} is empty — run `bggpipe resolve` first."
)
return session
raise typer.Exit(code=1)
session.run()
return session
+55 -19
View File
@@ -65,6 +65,13 @@
}
header kbd { background: rgba(255,255,255,.14); color: var(--paper); border-color: rgba(255,255,255,.25); }
main { max-width: 62rem; margin: 0 auto; padding: 1.4rem 1.2rem 6rem; }
#banner { max-width: 62rem; margin: 0 auto; padding: 0 1.2rem; }
.banner {
border-radius: 6px; padding: .6rem .9rem; margin-top: .9rem;
font-size: .85rem; line-height: 1.4;
}
.banner.error { background: #f6dcd6; border: 1px solid var(--reject); color: #6b2417; }
.banner.warn { background: var(--kraft); border: 1px solid var(--brass-deep); color: #6b5b33; }
h2 {
color: var(--paper);
font-family: "Iowan Old Style", Palatino, Georgia, serif;
@@ -236,6 +243,7 @@
<kbd>v</kbd> veto merge · <kbd>d</kbd> dismiss
</span>
</header>
<div id="banner"></div>
<main id="main"></main>
<script>
"use strict";
@@ -245,19 +253,36 @@ let active = 0;
const esc = s => String(s ?? "").replace(/[&<>"']/g,
c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
function showBanner(html) {
document.getElementById("banner").innerHTML = html;
}
function errorBanner(detail) {
showBanner(`<div class="banner error">couldn't load review state: ${esc(detail)}` +
` — is the server running? (check its terminal)</div>`);
}
async function refresh() {
STATE = await (await fetch("/api/state")).json();
const res = await fetch("/api/state");
if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => res.statusText)}`);
STATE = await res.json();
render();
}
async function post(url, body) {
const res = await fetch(url, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify(body),
});
let res;
try {
res = await fetch(url, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify(body),
});
} catch (err) {
alert("That didn't save (no response from the server): " + err);
return;
}
if (!res.ok) {
const detail = (await res.json()).detail || res.statusText;
const detail = await res.json().then(d => d.detail).catch(() => res.statusText);
alert("That didn't save: " + detail);
return;
}
@@ -298,7 +323,7 @@ function matchCard(row, idx) {
· rank ${esc(c.rank ?? "—")} · owned ${esc(c.owned ?? "—")}</span></span>
</li>`).join("");
return `
<section class="card actionable" data-kind="match" data-idx="${idx}"
<section class="card actionable" data-kind="match" data-rowix="${row.row_ix}"
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
${shots(row.photos)}
<div class="body">
@@ -324,7 +349,7 @@ function versionCard(row, idx) {
· ${esc((v.languages || []).join(", "))} · score ${esc(v.score ?? "—")}</span></span>
</li>`).join("");
return `
<section class="card actionable" data-kind="version" data-idx="${idx}"
<section class="card actionable" data-kind="version" data-rowix="${row.row_ix}"
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
<div class="body">
<p class="title">${esc(row.bgg_name || row.title_raw)}</p>
@@ -361,6 +386,8 @@ function ticket(s) {
function render() {
const m = document.getElementById("main");
const s = STATE;
showBanner((s.warnings || []).length
? `<div class="banner warn">${s.warnings.map(esc).join("<br>")}</div>` : "");
document.getElementById("tally").innerHTML =
`<span><b>${s.pending.length}</b> matches</span>
<span><b>${s.versions.length}</b> editions</span>
@@ -480,13 +507,14 @@ function highlight() {
if (el) el.scrollIntoView({block: "nearest", behavior: "auto"});
}
const rowIx = card => card.dataset.rowix === undefined ? null : Number(card.dataset.rowix);
const decide = (card, action, bgg_id = null) => post("/api/decision", {
title_raw: card.dataset.title, source_photos: card.dataset.photos,
action, bgg_id,
row_ix: rowIx(card), action, bgg_id,
});
const version = (card, action, version_id = null) => post("/api/version", {
title_raw: card.dataset.title, source_photos: card.dataset.photos,
action, version_id,
row_ix: rowIx(card), action, version_id,
});
const dismiss = t => post("/api/dismiss", {
photo: t.dataset.photo, location: t.dataset.location,
@@ -519,22 +547,30 @@ document.addEventListener("keydown", e => {
else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); }
});
refresh();
refresh().catch(err => errorBanner(err.message || err));
// Live-follow the data files: extract/resolve runs in another terminal show
// up on the next poll. Only re-render on an actual change (keeps the
// keyboard cursor stable) and never mid-typing in a manual-ID input.
let pollMisses = 0;
setInterval(async () => {
const el = document.activeElement;
if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA")) return;
let fresh = null;
try {
const fresh = await (await fetch("/api/state")).json();
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
STATE = fresh;
render();
}
} catch {
/* server restarting (--dev) — retry on the next tick */
const res = await fetch("/api/state");
if (!res.ok) throw new Error(`${res.status}`);
fresh = await res.json();
} catch (err) {
// transient during --dev restarts; persistent means the server is gone
if (++pollMisses >= 3) errorBanner(`lost contact (${err.message || err})`);
return;
}
if (pollMisses >= 3) showBanner(""); // recovered: clear the lost-contact banner
pollMisses = 0;
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
STATE = fresh;
render();
}
}, 3000);
</script>
+84 -27
View File
@@ -27,6 +27,7 @@ import os
import random
import re
import time
from collections import Counter
from collections.abc import Callable
from dataclasses import dataclass
from datetime import UTC, datetime
@@ -35,7 +36,7 @@ from typing import Protocol
import typer
from bggpipe.bgg_client import BGGAuthError, BGGClient
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
from bggpipe.config import Config
from bggpipe.models import CollectionItem
@@ -51,7 +52,7 @@ UPLOAD_LOG_COLUMNS = [
"timestamp",
"error",
]
DONE_STATUSES = {"added", "updated", "already_present"}
DONE_STATUSES = {"added", "added_no_version", "updated", "already_present"}
MAX_VERSION_PAGES = 40
@@ -73,7 +74,7 @@ class UploadJob:
return ("add", self.bgg_id, self.version_id)
def _row_key(row: dict) -> tuple[str, str, str]:
def _job_key(row: dict) -> tuple[str, str, str]:
if row["action"] == "update":
return ("update", row["collid"], "")
return ("add", row["bgg_id"], row["version_id"])
@@ -109,7 +110,7 @@ def build_queue(
done. Returns (jobs, skipped_done, skipped_failed)."""
latest: dict[tuple[str, str, str], str] = {}
for row in log_rows:
latest[_row_key(row)] = row["status"]
latest[_job_key(row)] = row["status"]
candidates = [
UploadJob(
@@ -134,15 +135,25 @@ def build_queue(
jobs: list[UploadJob] = []
skipped_done = skipped_failed = 0
deferred: list[UploadJob] = []
update_game_seen: set[str] = set()
for job in candidates:
status = latest.get(job.key)
if status in DONE_STATUSES:
skipped_done += 1
elif status == "failed" and not retry_failed:
skipped_failed += 1
elif job.action == "update" and job.bgg_id in update_game_seen:
# The row-edit flow finds rows by game name, not collid — a
# second same-game update this run could reopen the copy the
# first one just versioned and overwrite it. One per run; the
# next run (after --verify) picks up the rest.
deferred.append(job)
else:
if job.action == "update":
update_game_seen.add(job.bgg_id)
jobs.append(job)
return jobs, skipped_done, skipped_failed
return jobs, skipped_done, skipped_failed, deferred
def _scrub(text: str) -> str:
@@ -155,6 +166,12 @@ def _scrub(text: str) -> str:
return text
class LoginError(RuntimeError):
"""Authentication is broken (bad credentials, Cloudflare block, changed
login page). Systemic by definition: retrying per-game would hammer the
login endpoint and poison upload_log.csv with misleading failures."""
class Uploader(Protocol):
def add_game(self, job: UploadJob) -> tuple[str, str]: ...
@@ -215,11 +232,16 @@ class PlaywrightUploader:
return
page = self._page
self._goto(f"{BGG}/")
if "just a moment" in (page.title() or "").casefold():
raise LoginError(
"Cloudflare is challenging this browser ('Just a moment...') "
"— run headed (drop --headless) and click the widget once"
)
if self._signed_out():
user = os.environ.get("BGG_USERNAME", "")
password = os.environ.get("BGG_PASSWORD", "")
if not (user and password):
raise RuntimeError(
raise LoginError(
"BGG_USERNAME and BGG_PASSWORD env vars are required to log in"
)
self._goto(f"{BGG}/login")
@@ -229,7 +251,13 @@ class PlaywrightUploader:
page.locator("#inputUsername").fill(user)
page.locator("#inputPassword").fill(password)
page.get_by_role("button", name="Sign In").click()
page.wait_for_url(lambda url: "/login" not in url, timeout=120_000)
try:
page.wait_for_url(lambda url: "/login" not in url, timeout=120_000)
except self._timeout_error as err:
raise LoginError(
"login did not complete (wrong credentials, or the login "
"page changed — see docs/bgg-upload-flow.md)"
) from err
self._context.storage_state(path=str(self._storage_state))
self._authed = True
@@ -266,6 +294,7 @@ class PlaywrightUploader:
if nxt.count() == 0 or nxt.is_disabled():
break
nxt.click()
self._page.wait_for_timeout(2_000) # etiquette: paginating hits BGG too
# Two-level dismissal: the sub-view has its own Cancel distinct from
# the main dialog's.
dialog.get_by_role("button", name="Cancel").first.click()
@@ -283,13 +312,14 @@ class PlaywrightUploader:
"heading", name=re.compile(re.escape(job.name), re.I)
).wait_for(timeout=15_000)
dialog.get_by_label("Own").check()
note = ""
status, note = "added", ""
if job.version_name and not self._select_version(dialog, job.version_name):
status = "added_no_version"
note = f"version {job.version_name!r} not in picker; added without version"
dialog.get_by_role("button", name="Save").click()
# The dialog is hidden after save, not removed from the DOM.
dialog.wait_for(state="hidden", timeout=15_000)
return "added", note
return status, note
def update_entry(self, job: UploadJob) -> tuple[str, str]:
"""Set the version on an EXISTING entry — strictly additive.
@@ -332,6 +362,7 @@ def _process(
now: Callable[[], str],
) -> list[dict]:
results = []
consecutive: tuple[str, int] = ("", 0)
for i, job in enumerate(jobs):
if i:
sleep(rng.uniform(2.0, 4.0)) # polite pacing between games (spec)
@@ -340,6 +371,12 @@ def _process(
status, note = uploader.add_game(job)
else:
status, note = uploader.update_entry(job)
except LoginError as exc:
# Systemic: every remaining job would fail identically. Abort
# WITHOUT logging failures, so the next run just retries.
typer.echo(f" aborting — login is broken: {_scrub(str(exc))}")
typer.echo(f" {len(jobs) - i} job(s) left untouched for the next run.")
break
except Exception as exc: # per-game isolation: log it, keep going
status, note = "failed", _scrub(f"{type(exc).__name__}: {exc}")
row = {
@@ -350,12 +387,25 @@ def _process(
"version_id": job.version_id,
"status": status,
"timestamp": now(),
"error": note if status == "failed" else note,
# the column carries degradation notes on successes too
# (e.g. added_no_version), not just failure text
"error": note,
}
append_log_row(log_path, row)
results.append(row)
suffix = f"{note}" if note else ""
typer.echo(f" {job.name}: {status}{suffix}")
if status == "failed":
consecutive = (note, consecutive[1] + 1 if note == consecutive[0] else 1)
if consecutive[1] >= 3:
typer.echo(
" aborting — 3 identical consecutive failures look "
"systemic, not per-game; remaining jobs left for the "
"next run"
)
break
else:
consecutive = ("", 0)
return results
@@ -370,14 +420,18 @@ def verify_uploads(log_rows: list[dict], collection: list[CollectionItem]) -> li
problems = []
latest: dict[tuple[str, str, str], dict] = {}
for row in log_rows:
latest[_row_key(row)] = row
latest[_job_key(row)] = row
for row in latest.values():
if row["status"] == "added":
if row["status"] in ("added", "added_no_version"):
copies = by_object.get(int(row["bgg_id"]), [])
if not copies:
problems.append(f"{row['name']}: logged added but not in collection")
elif row["version_id"] and not any(
str(c.version_id or "") == row["version_id"] for c in copies
elif (
row["status"] == "added" # no_version: absence is expected
and row["version_id"]
and not any(
str(c.version_id or "") == row["version_id"] for c in copies
)
):
problems.append(
f"{row['name']}: in collection but no copy has "
@@ -419,11 +473,7 @@ def run_upload(
# cache marker travels with the stub XML (gitignored, so a fresh clone
# loses it), while data/STUB_DATA.marker is COMMITTED alongside the
# stub-derived CSVs — so a clone can never upload placeholder ids.
markers = [
cfg.cache_dir / "STUB_FIXTURES.marker",
cfg.data_dir / "STUB_DATA.marker",
]
marker = next((m for m in markers if m.exists()), None)
marker = next((m for m in cfg.stub_marker_paths if m.exists()), None)
if marker is not None:
if not dry_run:
typer.echo(
@@ -439,12 +489,15 @@ def run_upload(
"placeholders, not real BGG data.\n"
)
log_path = cfg.data_dir / "upload_log.csv"
to_add = _read_csv(cfg.data_dir / "to_add.csv")
to_update = _read_csv(cfg.data_dir / "to_update.csv")
log_path = cfg.upload_log_path
if not cfg.to_add_path.exists():
typer.echo(f"{cfg.to_add_path} not found — run `bggpipe diff` first.")
raise typer.Exit(code=1)
to_add = _read_csv(cfg.to_add_path)
to_update = _read_csv(cfg.to_update_path)
log_rows = _read_csv(log_path)
jobs, skipped_done, skipped_failed = build_queue(
jobs, skipped_done, skipped_failed, deferred = build_queue(
to_add, to_update, log_rows, retry_failed=retry_failed
)
if limit is not None:
@@ -456,6 +509,12 @@ def run_upload(
f" (skipping {skipped_done} already done, {skipped_failed} "
"previously failed — use --retry-failed)"
)
if deferred:
typer.echo(
f"{len(deferred)} version update(s) deferred: one update per game "
"per run (the row-edit flow can't target a collid) — re-run "
"upload after --verify confirms this batch."
)
results: list[dict] = []
if dry_run:
@@ -484,9 +543,7 @@ def run_upload(
results = _process(real, jobs, log_path, sleep=sleep, rng=rng, now=now)
else:
results = _process(uploader, jobs, log_path, sleep=sleep, rng=rng, now=now)
counts: dict[str, int] = {}
for row in results:
counts[row["status"]] = counts.get(row["status"], 0) + 1
counts = Counter(row["status"] for row in results)
summary = " · ".join(f"{n} {status}" for status, n in sorted(counts.items()))
typer.echo(f"\n{summary or 'nothing to do'}")
else:
@@ -501,7 +558,7 @@ def _run_verify(cfg: Config, client: BGGClient | None, log_path: Path) -> None:
if not cfg.bgg_username:
typer.echo("--verify needs BGG_USERNAME in the environment.")
return
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
client = client or client_for(cfg)
try:
collection = client.collection_full(cfg.bgg_username, refresh=True)
except BGGAuthError as exc:
+51 -21
View File
@@ -15,6 +15,8 @@ from __future__ import annotations
import io
import json
import os
import threading
from collections import Counter
from importlib import resources
from pathlib import Path
@@ -26,11 +28,10 @@ from pydantic import BaseModel
from rich.console import Console
from bggpipe.bgg_client import BGGClient
from bggpipe.config import DEFAULT_REVIEW_PORT as DEFAULT_PORT # single home
from bggpipe.config import Config
from bggpipe.review import ReviewSession
DEFAULT_PORT = 8377
def load_thumbnails(cache_dir: Path) -> dict[int, str]:
"""bgg_id -> thumbnail URL, from cached /thing XML only (no live calls).
@@ -41,7 +42,7 @@ def load_thumbnails(cache_dir: Path) -> dict[int, str]:
for path in cache_dir.glob("thing_*.xml"):
try:
root = _safe_fromstring(path.read_text())
except Exception: # noqa: BLE001 — a corrupt cache file must not kill the UI
except Exception: # a corrupt cache file must not kill the UI
continue
for item in root.findall("item"):
thumb = (item.findtext("thumbnail") or "").strip()
@@ -80,6 +81,7 @@ class DismissStore:
class DecisionBody(BaseModel):
title_raw: str
source_photos: str
row_ix: int | None = None
action: str # "pick" | "manual" | "reject"
bgg_id: int | None = None
@@ -87,6 +89,7 @@ class DecisionBody(BaseModel):
class VersionBody(BaseModel):
title_raw: str
source_photos: str
row_ix: int | None = None
action: str # "pick" | "unknown"
version_id: int | None = None
@@ -94,6 +97,7 @@ class VersionBody(BaseModel):
class VetoBody(BaseModel):
title_raw: str
source_photos: str
row_ix: int | None = None
class DismissBody(BaseModel):
@@ -112,7 +116,11 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
client=client,
)
thumbnails = load_thumbnails(cfg.cache_dir)
dismissed = DismissStore(cfg.data_dir / "unidentified_dismissed.json")
dismissed = DismissStore(cfg.dismissed_path)
# FastAPI runs sync endpoints in a threadpool: without this, a polled
# GET /api/state can freshen()-swap session.rows out from under a
# concurrent decision POST, silently dropping the decision.
lock = threading.Lock()
def freshen() -> None:
"""Serve every request from the current file state: an extract or
@@ -121,8 +129,16 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
if session.reload_if_changed():
thumbnails = load_thumbnails(cfg.cache_dir)
def find_row(title_raw: str, source_photos: str) -> dict:
def find_row(
title_raw: str, source_photos: str, row_ix: int | None = None
) -> dict:
freshen()
# ordinal first: (title_raw, source_photos) is not unique when one
# photo holds two editions of the same game
if row_ix is not None and 0 <= row_ix < len(session.rows):
row = session.rows[row_ix]
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
return row
for row in session.rows:
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
return row
@@ -134,12 +150,13 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
return {p.name for p in cfg.photos_dir.iterdir() if p.is_file()}
def row_payload(row: dict) -> dict:
entry = session.cues_for(row["title_raw"])
entry = session.cues_for(row["title_raw"], row["source_photos"])
available = photo_names()
candidates = json.loads(row["candidates_json"] or "[]")
for c in candidates:
c["thumbnail"] = thumbnails.get(c.get("bgg_id"))
return {
"row_ix": session.rows.index(row),
"title_raw": row["title_raw"],
"source_photos": row["source_photos"],
"match_status": row["match_status"],
@@ -156,6 +173,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
def version_payload(row: dict) -> dict:
return {
"row_ix": session.rows.index(row),
"title_raw": row["title_raw"],
"source_photos": row["source_photos"],
"bgg_name": row["bgg_name"],
@@ -164,9 +182,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
def state() -> dict:
freshen()
counts: dict[str, int] = {}
for row in session.rows:
counts[row["match_status"]] = counts.get(row["match_status"], 0) + 1
counts = Counter(row["match_status"] for row in session.rows)
resolved_titles = {r["title_raw"] for r in session.rows}
rows_by_title: dict[str, dict] = {}
for r in session.rows:
@@ -216,6 +232,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
for r in session.merged_rows()
]
return {
"warnings": session.warnings[-10:],
"pending": [row_payload(r) for r in session.pending_rows()],
"versions": [version_payload(r) for r in session.version_rows()],
"merges": merges,
@@ -240,11 +257,16 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.get("/api/state")
def api_state() -> dict:
return state()
with lock:
return state()
@app.post("/api/decision")
def api_decision(body: DecisionBody) -> dict:
row = find_row(body.title_raw, body.source_photos)
with lock:
return _decide(body)
def _decide(body: DecisionBody) -> dict:
row = find_row(body.title_raw, body.source_photos, body.row_ix)
if body.action == "pick":
candidates = json.loads(row["candidates_json"] or "[]")
chosen = next(
@@ -265,7 +287,11 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.post("/api/version")
def api_version(body: VersionBody) -> dict:
row = find_row(body.title_raw, body.source_photos)
with lock:
return _version(body)
def _version(body: VersionBody) -> dict:
row = find_row(body.title_raw, body.source_photos, body.row_ix)
if body.action == "unknown":
session.decide_version(row, None)
elif body.action == "pick":
@@ -279,19 +305,23 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.post("/api/veto-merge")
def api_veto_merge(body: VetoBody) -> dict:
row = find_row(body.title_raw, body.source_photos)
if row["match_status"] != "merged":
raise HTTPException(400, "row is not merged")
session.veto_merge(row)
return state()
with lock:
row = find_row(body.title_raw, body.source_photos, body.row_ix)
if row["match_status"] != "merged":
raise HTTPException(400, "row is not merged")
session.veto_merge(row)
return state()
@app.post("/api/dismiss")
def api_dismiss(body: DismissBody) -> dict:
dismissed.add(_sighting_key(body.photo, body.model_dump(exclude={"photo"})))
return state()
with lock:
dismissed.add(
_sighting_key(body.photo, body.model_dump(exclude={"photo"}))
)
return state()
@app.get("/photos/{name}")
def photo(name: str):
def photo(name: str) -> FileResponse:
if name not in photo_names(): # also blocks any path traversal
raise HTTPException(404, "no such photo")
return FileResponse(cfg.photos_dir / name)
@@ -303,7 +333,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
}
@app.get("/static/{name}")
def static_asset(name: str):
def static_asset(name: str) -> Response:
media_type = _STATIC.get(name) # allowlist: no traversal possible
if media_type is None:
raise HTTPException(404, "no such asset")
+51
View File
@@ -131,3 +131,54 @@ def test_401_raises_actionable_auth_error(tmp_path, monkeypatch):
client, _, _ = make_client(tmp_path, [(401, "Unauthorized")])
with pytest.raises(BGGAuthError, match="BGG_API_TOKEN"):
client.get_xml("search", {"query": "catan"})
# -- audit-fix regressions ----------------------------------------------
def test_http_200_error_document_raises_and_is_never_cached(tmp_path):
# BGG serves some errors as HTTP 200 <errors> XML; caching one would
# poison every future run for that query
from bggpipe.models import BGGResponseError
errors_xml = "<errors><error><message>Invalid username</message></error></errors>"
client, _, _ = make_client(tmp_path, [(200, errors_xml)])
with pytest.raises(BGGResponseError):
client.get_xml("collection", {"username": "nobody", "own": "1"})
assert list((tmp_path / "cache").glob("*.xml")) == []
def test_collection_full_merges_and_dedupes_by_collid(tmp_path):
base_xml = (
'<items totalitems="2">'
'<item objectid="13" collid="100" subtype="boardgame">'
'<name>Catan</name><status own="1"/></item>'
'<item objectid="177" collid="101" subtype="boardgame">'
'<name>Advanced Civilization</name><status own="1"/></item>'
"</items>"
)
expansion_xml = (
'<items totalitems="1">'
'<item objectid="177" collid="101" subtype="boardgameexpansion">'
'<name>Advanced Civilization</name><status own="1"/></item>'
"</items>"
)
client, _, _ = make_client(tmp_path, [(200, base_xml), (200, expansion_xml)])
items = client.collection_full("eric")
assert len(items) == 2 # collid 101 appears in both responses: one copy
assert {i.coll_id for i in items} == {100, 101}
def test_collection_item_missing_collid_refuses_to_parse(tmp_path):
# a truncated response must fail loudly, not coerce ids to 0 and let
# the dedupe silently drop owned games
from bggpipe.models import BGGResponseError, parse_collection
bad_xml = (
'<items totalitems="1">'
'<item objectid="13" subtype="boardgame">'
'<name>Catan</name><status own="1"/></item>'
"</items>"
)
with pytest.raises(BGGResponseError):
parse_collection(bad_xml)
+15
View File
@@ -1,3 +1,7 @@
"""Config loading: toml knobs, env-only username, unknown-key warning."""
from __future__ import annotations
from pathlib import Path
from bggpipe.config import Config, load_config
@@ -29,3 +33,14 @@ def test_username_comes_from_env_only(tmp_path, monkeypatch):
assert load_config(p).bgg_username == "from_env"
monkeypatch.delenv("BGG_USERNAME")
assert load_config(p).bgg_username == ""
def test_unknown_toml_keys_warn(tmp_path, monkeypatch):
# a typo'd knob must not silently fall back to defaults
import pytest
monkeypatch.delenv("BGG_USERNAME", raising=False)
p = tmp_path / "config.toml"
p.write_text('photo_dir = "oops"\n')
with pytest.warns(UserWarning, match="photo_dir"):
load_config(p)
+64 -5
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
from pathlib import Path
from bggpipe.config import Config
from bggpipe.diff import compute_diff, load_snapshot_collection
from bggpipe.models import CollectionItem
@@ -114,7 +115,10 @@ def test_owned_with_matching_version_is_just_owned():
assert not result.to_update and not result.to_add
def test_owned_with_different_version_reports_disagreement_untouched():
def test_confident_version_matching_no_copy_is_a_second_copy_to_add():
# Spec: a (bgg_id, version_id) pair is owned only if a collection item
# matches BOTH. All copies carry different versions -> this is an
# additional physical copy; existing entries are never edited.
result = compute_diff(
[
_match(
@@ -127,9 +131,10 @@ def test_owned_with_different_version_reports_disagreement_untouched():
],
[_item(266192, 5, version_id=465063)],
)
assert result.already_owned == ["Wingspan"]
assert not result.to_update # additive only: never edit a set version
assert "fourth printing" in result.disagreements[0]
assert [r["version_id"] for r in result.to_add] == ["521212"]
assert "fourth printing" in result.second_copies[0]
assert result.already_owned == []
def test_version_unknown_owned_by_bare_id():
@@ -169,8 +174,11 @@ def test_pending_rejected_and_unseen_are_reported():
def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
matches = [
_match("Joking Hazard", "193621"),
{**_match("Jokin Ha...", "193621", status="merged"),
"merged_into": "Joking Hazard", "source_photos": "other.jpg"},
{
**_match("Jokin Ha...", "193621", status="merged"),
"merged_into": "Joking Hazard",
"source_photos": "other.jpg",
},
]
result = compute_diff(matches, []) # empty collection -> to_add
assert result.merged == 1
@@ -178,3 +186,54 @@ def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
(row,) = result.to_add
assert row["title_raw"] == "Joking Hazard"
assert row["source_photos"] == "other.jpg;x.jpg" # combined
def test_versionless_copies_exhaust_then_second_copy_becomes_add():
# two confident-version matches, ONE versionless copy: the first consumes
# it (to_update), the second is an additional physical copy (to_add)
result = compute_diff(
[
_match("Sorcerer", "39", vstatus="version_auto", vid="111", vname="1st"),
_match("Sorcerer", "39", vstatus="version_auto", vid="222", vname="2nd"),
],
[_item(39, 701)],
)
assert [u["version_id"] for u in result.to_update] == ["111"]
assert [a["version_id"] for a in result.to_add] == ["222"]
assert len(result.second_copies) == 1
def test_run_diff_outputs_feed_upload_unchanged(tmp_path, monkeypatch):
# the cross-stage contract: whatever run_diff writes, run_upload must
# read — a column rename on either side has to fail HERE
import shutil
from bggpipe.diff import run_diff
from bggpipe.resolve import write_matches
from bggpipe.upload import run_upload
monkeypatch.delenv("BGG_API_TOKEN", raising=False)
cfg = Config(data_dir=tmp_path)
fixtures = Path(__file__).parent / "fixtures"
for name in (
"collection_snapshot_base.xml",
"collection_snapshot_expansions.xml",
):
shutil.copy(fixtures / name, tmp_path / name)
write_matches(
cfg.matches_path,
[
_match("Wingspan", "266192", status="auto"), # not in the snapshots
_match("5 MINUTE DUNGEON", "207830", status="auto"), # owned
],
)
result = run_diff(cfg)
assert [r["bgg_id"] for r in result.to_add] == ["266192"]
from test_upload import FakeUploader
fake = FakeUploader()
run_upload(cfg, uploader=fake, sleep=lambda s: None, now=lambda: "t")
assert [(j.action, j.bgg_id, j.name) for j in fake.calls] == [
("add", "266192", "Wingspan")
]
+5 -5
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import base64
import io
import json
import pytest
import typer
@@ -83,21 +84,21 @@ def test_parse_object_with_titles_and_unidentified():
"unidentified": [{"location": "top shelf, left of Catan",
"partial_text": "WAR", "art_notes": "red spine"}]}
```"""
titles, unidentified = parse_vision_response(text)
titles, unidentified, _ = parse_vision_response(text)
assert titles[0]["title_raw"] == "Catan"
assert unidentified[0]["location"] == "top shelf, left of Catan"
def test_parse_legacy_bare_array_still_works():
text = '```json\n[{"title_raw": "Catan", "confidence": "high"}]\n```'
titles, unidentified = parse_vision_response(text)
titles, unidentified, _ = parse_vision_response(text)
assert titles[0]["title_raw"] == "Catan"
assert unidentified == []
def test_parse_tolerates_prose_around_json():
text = 'Here are the games:\n[{"title_raw": "Wingspan"}]\nLet me know!'
titles, _ = parse_vision_response(text)
titles, _, _ = parse_vision_response(text)
assert titles[0]["title_raw"] == "Wingspan"
@@ -106,7 +107,7 @@ def test_parse_drops_malformed_entries():
'{"titles": [{"title_raw": "Catan"}, {"no_title": true}, "just a string"],'
' "unidentified": [{}, "not a dict", {"location": "somewhere"}]}'
)
titles, unidentified = parse_vision_response(text)
titles, unidentified, _ = parse_vision_response(text)
assert len(titles) == 1
assert len(unidentified) == 1 # empty {} and the bare string are dropped
@@ -227,7 +228,6 @@ def test_run_extract_empty_photos_dir_exits(tmp_path):
# -- unidentified sightings ---------------------------------------------
import json # noqa: E402
OBJECT_PAYLOAD = json.dumps(
{
+4
View File
@@ -1,3 +1,7 @@
"""XML parser tests against hand-built API2 response shapes."""
from __future__ import annotations
import pytest
from bggpipe.models import (
+4
View File
@@ -1,3 +1,7 @@
"""Title normalization must be symmetric and aggressive (spec)."""
from __future__ import annotations
from bggpipe.normalize import normalize_title
+120 -12
View File
@@ -14,17 +14,23 @@ from pathlib import Path
import httpx
import pytest
from bggpipe.bgg_client import BGGClient
from bggpipe.bgg_client import BGGClient, cache_key
from bggpipe.config import Config
from bggpipe.models import GameVersion
from bggpipe.normalize import normalize_title
from bggpipe.resolve import (
Candidate,
MatchRow,
TitleEntry,
_dominant,
_score_version,
_truncation_heads,
dedupe_matches,
load_titles,
read_matches,
resolve_entry,
run_resolve,
write_matches,
)
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
@@ -191,9 +197,6 @@ def test_score_version_no_overlap():
# -- progressive title truncation (long transcribed box titles) ---------
from bggpipe.normalize import normalize_title # noqa: E402
from bggpipe.resolve import _truncation_heads # noqa: E402
CIV_TITLE = (
"CIVILIZATION Game of the Heroic Age - The Dawn of History 8000 BC to 250 BC"
)
@@ -347,13 +350,15 @@ def test_run_resolve_saves_progress_when_token_missing(tmp_path):
# -- post-resolve dedupe ------------------------------------------------
from bggpipe.bgg_client import cache_key as _cache_key # noqa: E402
from bggpipe.resolve import dedupe_matches # noqa: E402
def _mrow(
title, bgg_id, photos, name="Joking Hazard",
vstatus="version_unknown", vid="", status="auto",
title,
bgg_id,
photos,
name="Joking Hazard",
vstatus="version_unknown",
vid="",
status="auto",
):
return {
"title_raw": title,
@@ -446,8 +451,6 @@ def test_dedupe_is_idempotent_and_skips_merged():
def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
from bggpipe.resolve import write_matches as _wm # noqa: F401
cache = tmp_path / "cache"
cache.mkdir()
wingspan_xml = (
@@ -456,7 +459,7 @@ def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
"</item></items>"
)
for query in ("Wingspan", "WINGSPAN!"):
key = _cache_key(
key = cache_key(
"search", {"query": query, "type": "boardgame,boardgameexpansion"}
)
(cache / key).write_text(wingspan_xml)
@@ -482,3 +485,108 @@ def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
assert saved["Wingspan"]["match_status"] == "auto"
assert saved["WINGSPAN!"]["match_status"] == "merged"
assert saved["WINGSPAN!"]["merged_into"] == "Wingspan"
# -- audit-fix regressions ----------------------------------------------
def test_run_resolve_force_rebuilds_from_scratch(client, tmp_path):
data_dir = tmp_path / "data"
data_dir.mkdir()
shutil.copy(TITLES_JSON, data_dir / "titles.json")
cfg = Config(data_dir=data_dir)
run_resolve(cfg, client=client)
# poison one row: force must throw it away and re-resolve everything
rows = read_matches(cfg.matches_path)
rows[0]["match_status"] = "rejected"
write_matches(cfg.matches_path, rows)
forced = run_resolve(cfg, force=True, client=client)
assert len(forced) == 7 # every title re-resolved, none skipped
fresh = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
assert fresh["Catan"]["match_status"] == "auto"
def test_new_photo_of_resolved_game_updates_row_instead_of_duplicating(
client, tmp_path
):
data_dir = tmp_path / "data"
data_dir.mkdir()
shutil.copy(TITLES_JSON, data_dir / "titles.json")
cfg = Config(data_dir=data_dir)
run_resolve(cfg, client=client)
n_rows = len(read_matches(cfg.matches_path))
# extract sees Catan again on a reshoot photo: the entry's photo set
# grows, its (title, photos) key changes
titles = json.loads((data_dir / "titles.json").read_text())
for entry in titles:
if entry["title_raw"] == "Catan":
entry["source_photos"] = sorted([*entry["source_photos"], "reshoot.jpg"])
(data_dir / "titles.json").write_text(json.dumps(titles))
assert run_resolve(cfg, client=client) == [] # nothing re-resolved
rows = read_matches(cfg.matches_path)
assert len(rows) == n_rows # and no duplicate row appended
catan = next(r for r in rows if r["title_raw"] == "Catan")
assert "reshoot.jpg" in catan["source_photos"] # provenance followed
def test_dedupe_never_overturns_a_human_veto():
a = _mrow("CATAN", "13", "p1.jpg")
b = _mrow("Catan", "13", "p2.jpg", status="approved")
b["dedupe_veto"] = "1" # review said: genuinely two copies
events = dedupe_matches([a, b], [])
assert events == []
assert b["match_status"] == "approved"
def test_publisher_pick_refuses_multiple_same_publisher_candidates():
from bggpipe.resolve import _publisher_pick
entry = TitleEntry(
title_raw="Sorcerer", title_normalized="sorcerer", publisher_hint="SPI"
)
cands = [
_cand(1, exact=True),
_cand(2, exact=True),
]
for c in cands:
c.publishers = ["Simulations Publications, Inc. (SPI)"]
assert _publisher_pick(entry, cands) is None
def test_publisher_pick_refuses_mixed_base_and_expansion():
from bggpipe.resolve import _publisher_pick
entry = TitleEntry(
title_raw="Wingspan", title_normalized="wingspan", publisher_hint="Stonemaier"
)
base = _cand(1, exact=True, type_="boardgame")
expansion = _cand(2, exact=True, type_="boardgameexpansion")
for c in (base, expansion):
c.publishers = ["Stonemaier Games"]
assert _publisher_pick(entry, [base, expansion]) is None
def test_resolve_version_handles_unknown_id():
from bggpipe.resolve import resolve_version
class EmptyThings:
def things(self, ids, **kwargs):
return []
entry = TitleEntry(title_raw="X", title_normalized="x", publisher_hint="Someone")
row = MatchRow(title_raw="X", bgg_id=999999)
resolve_version(EmptyThings(), entry, row) # must not raise
assert row.version_status == "version_unknown"
def test_empty_normalized_title_never_matches(client):
# 风声 normalizes to "" — empty-vs-empty must not count as exact
from bggpipe.resolve import _plausible_candidates
entry = TitleEntry(title_raw="风声", title_normalized="")
# any cached query works; candidates must be rejected regardless of name
assert _plausible_candidates(client, entry, "Catan") == []
+95 -1
View File
@@ -9,12 +9,13 @@ import json
from pathlib import Path
import httpx
import pytest
from rich.console import Console
from bggpipe.bgg_client import BGGClient
from bggpipe.config import Config
from bggpipe.resolve import read_matches, write_matches
from bggpipe.review import run_review
from bggpipe.review import ReviewSession, run_review
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
@@ -288,3 +289,96 @@ def test_version_pass_is_skippable(tmp_path):
)
(row,) = read_matches(cfg.matches_path)
assert row["version_status"] == "version_ambiguous" # untouched, review later
# -- audit-fix regressions ----------------------------------------------
def test_veto_merge_persists_against_future_dedupe(tmp_path):
from bggpipe.resolve import read_matches
cfg = _setup(
tmp_path,
[
_row(title_raw="CATAN", match_status="merged", merged_into="Catan"),
_row(title_raw="Catan", match_status="auto", bgg_id="13"),
],
)
session = ReviewSession(
cfg,
console=quiet_console(),
input_fn=scripted(),
client=unauthorized_client(tmp_path),
)
merged = next(r for r in session.rows if r["match_status"] == "merged")
session.veto_merge(merged)
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
assert saved["CATAN"]["match_status"] == "approved"
assert saved["CATAN"]["dedupe_veto"] == "1" # survives resolve re-runs
def test_failed_save_never_leaves_memory_ahead_of_disk(tmp_path, monkeypatch):
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
session = ReviewSession(
cfg,
console=quiet_console(),
input_fn=scripted(),
client=unauthorized_client(tmp_path),
)
row = session.rows[0]
import bggpipe.review as review_mod
def exploding_write(path, rows):
raise OSError("disk full")
monkeypatch.setattr(review_mod, "write_matches", exploding_write)
with pytest.raises(OSError):
session.decide_reject(row)
# memory was rolled back to what disk actually holds
assert session.rows[0]["match_status"] == "unmatched"
assert session.decisions == 0
def test_save_merges_own_decision_over_concurrent_external_rewrite(tmp_path):
from bggpipe.resolve import read_matches, write_matches
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
session = ReviewSession(
cfg,
console=quiet_console(),
input_fn=scripted(),
client=unauthorized_client(tmp_path),
)
row = session.rows[0]
# resolve appends a new row in another terminal AFTER our session loaded
external = read_matches(cfg.matches_path)
external.append(_row(title_raw="Newcomer", match_status="auto", bgg_id="7"))
write_matches(cfg.matches_path, external)
session.decide_reject(row)
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
assert saved["Mystery"]["match_status"] == "rejected" # our decision
assert "Newcomer" in saved # their row survived too
def test_manual_id_unknown_to_bgg_warns_instead_of_crashing(tmp_path):
# an empty /thing result (mistyped id) used to crash the whole session
import httpx as _httpx
empty_things = BGGClient(
cache_dir=tmp_path / "cache",
transport=_httpx.MockTransport(
lambda req: _httpx.Response(200, text='<items total="0"></items>')
),
sleep=lambda s: None,
)
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
session = ReviewSession(
cfg, console=quiet_console(), input_fn=scripted(), client=empty_things
)
session.decide_manual(session.rows[0], 999999)
assert session.rows[0]["match_status"] == "approved"
assert session.rows[0]["bgg_id"] == "999999"
assert any("no game with id 999999" in w for w in session.warnings)
+88 -5
View File
@@ -111,7 +111,7 @@ class FakeUploader:
def test_build_queue_skips_logged_successes():
jobs, done, failed = build_queue(
jobs, done, failed, _ = build_queue(
[_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")],
[_update_row(collid="9")],
[
@@ -127,7 +127,7 @@ def test_build_queue_skips_logged_successes():
def test_build_queue_second_copy_is_a_distinct_job():
# Same game, different version: a separate physical copy, so a
# logged add of one version must not swallow the other.
jobs, done, _ = build_queue(
jobs, done, _, _ = build_queue(
[
_add_row(bgg_id="1", version_id="10", version_name="First ed."),
_add_row(bgg_id="1", version_id="11", version_name="Second ed."),
@@ -141,9 +141,11 @@ def test_build_queue_second_copy_is_a_distinct_job():
def test_build_queue_failures_need_retry_flag():
log = [_log_row(action="add", bgg_id="1", status="failed")]
jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log)
jobs, _, skipped, _ = build_queue([_add_row(bgg_id="1")], [], log)
assert jobs == [] and skipped == 1
jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
jobs, _, skipped, _ = build_queue(
[_add_row(bgg_id="1")], [], log, retry_failed=True
)
assert len(jobs) == 1 and skipped == 0
@@ -153,7 +155,7 @@ def test_build_queue_latest_log_entry_wins():
_log_row(action="add", bgg_id="1", status="failed"),
_log_row(action="add", bgg_id="1", status="added"),
]
jobs, done, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
jobs, done, _, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
assert jobs == [] and done == 1
@@ -300,3 +302,84 @@ def test_fresh_clone_marker_blocks_upload_without_cache_dir(tmp_path):
_seed_data(tmp_path, to_add=[_add_row()])
with pytest.raises(typer.Exit):
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
# -- audit-fix regressions ----------------------------------------------
def test_login_error_aborts_without_poisoning_the_log(tmp_path):
from bggpipe.upload import LoginError
class BrokenLogin(FakeUploader):
def add_game(self, job):
raise LoginError("Cloudflare is challenging this browser")
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 4)])
results = run_upload(cfg, uploader=BrokenLogin(), sleep=lambda s: None, now=NOW)
assert results == [] # nothing logged: next run retries everything
assert not (tmp_path / "upload_log.csv").exists()
def test_three_identical_failures_abort_as_systemic(tmp_path):
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 6)])
fake = FakeUploader(failures={"Wingspan"}) # every job shares the name
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
assert len(results) == 3 # aborted after the third identical failure
logged = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
assert len(logged) == 3 # jobs 4-5 left unlogged and retryable
def test_added_no_version_is_done_and_verify_tolerates_it(tmp_path):
class NoVersionPicker(FakeUploader):
def add_game(self, job):
self.calls.append(job)
return "added_no_version", "version not in picker; added without version"
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row(version_id="99", version_name="4th ed.")])
run_upload(cfg, uploader=NoVersionPicker(), sleep=lambda s: None, now=NOW)
# done: re-running must NOT re-add (a duplicate collection entry)
again = FakeUploader()
assert run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW) == []
assert again.calls == []
# verify: game present without the version is the EXPECTED outcome
log = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
assert verify_uploads(log, [_item(1, 10)]) == []
def test_missing_to_add_csv_is_a_loud_precondition_failure(tmp_path):
cfg = _cfg(tmp_path) # no diff outputs seeded at all
with pytest.raises(typer.Exit):
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
def test_second_update_for_same_game_is_deferred(tmp_path):
# the row-edit flow can't target a collid, so only one update per game
# per run is safe
cfg = _cfg(tmp_path)
_seed_data(
tmp_path,
to_update=[
_update_row(collid="9", bgg_id="2"),
_update_row(collid="10", bgg_id="2", vid="26", vname="2nd ed."),
],
)
fake = FakeUploader()
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
assert [r["collid"] for r in results] == ["9"]
# after the first lands, the next run picks up the deferred one
again = FakeUploader()
results = run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW)
assert [j.collid for j in again.calls] == ["10"]
def test_real_run_without_credentials_exits_before_any_browser(tmp_path, monkeypatch):
monkeypatch.delenv("BGG_USERNAME", raising=False)
monkeypatch.delenv("BGG_PASSWORD", raising=False)
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row()])
with pytest.raises(typer.Exit):
run_upload(cfg, sleep=lambda s: None, now=NOW) # uploader=None: real path
assert not (tmp_path / "upload_log.csv").exists()
+20
View File
@@ -383,3 +383,23 @@ def test_own_saves_do_not_count_as_external_changes(tmp_path):
write_matches(cfg.matches_path, session.rows + [_row(title_raw="X")])
assert session.reload_if_changed() is True # someone else's
assert any(r["title_raw"] == "X" for r in session.rows)
def test_session_warnings_surface_in_state(tmp_path):
web, cfg = make_client(tmp_path)
# a manual id triggers a lookup against the 401-ing client: the session
# degrades and the warning must reach the payload (the session console
# is a StringIO here — this is the only way the user ever sees it)
res = web.post(
"/api/decision",
json={
"title_raw": "Mystery",
"source_photos": "shelf.jpg",
"action": "manual",
"bgg_id": 42,
},
)
assert res.status_code == 200
warnings = res.json()["warnings"]
assert any("couldn't look up id 42" in w for w in warnings)
assert warnings == web.get("/api/state").json()["warnings"]