Audit: 5-reviewer sweep — 19 fixes across every stage, +24 tests

Correctness: review vetoes persist via a dedupe_veto column (resolve
re-runs no longer overturn humans); diff emits second copies whose
confident version matches no owned copy (spec: pairs own only on both
ids) and fetches the live collection with refresh; resolve pairs
titles.json entries to rows by title so a reshoot photo updates
provenance instead of duplicating rows; version lookups survive empty
/thing results; publisher tie-break now honors the mixed
base/expansion veto and refuses multi-candidate picks; empty-normalized
(non-Latin) titles never count as exact.

Upload: LoginError aborts a run instead of logging N bogus failures
(and 3 identical consecutive failures abort as systemic); Cloudflare
interstitials are detected; added-without-version gets its own logged
status that verify understands; same-game updates run one per pass so
the name-targeted row edit can't overwrite a fresh version; absent
diff outputs fail loudly; pagination clicks are paced.

Web review: a lock serializes freshen/decide (threadpool race dropped
decisions); failed saves roll memory back and always alert the browser
(non-JSON 500s included); session warnings reach the page instead of a
StringIO; state-load failures and dead servers show banners instead of
a blank page; duplicate (title, photos) rows are addressable by
ordinal.

Consistency: shared CONFIDENT_VERSION_STATUSES, client_for(),
Config paths for every artifact, one review-port constant, named
matching thresholds, strict collection-id parsing, error-doc responses
never cached, unknown config keys warn, extract reports dropped vision
entries, fixture generators share escaping + marker text.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Eric Wagoner
2026-08-02 14:10:57 -04:00
parent abf7181475
commit 38e20f2c30
26 changed files with 1003 additions and 249 deletions
+1 -1
View File
@@ -21,7 +21,7 @@ Full design lives in `bgg-shelf-pipeline-spec.md` (read it before changing pipel
- `uv sync` — install deps (Python 3.12+, managed by **uv**; use `uv add`, never pip). Playwright needs a one-time `uv run playwright install chromium`. - `uv sync` — install deps (Python 3.12+, managed by **uv**; use `uv add`, never pip). Playwright needs a one-time `uv run playwright install chromium`.
- `uv run bggpipe <stage>` — run a pipeline stage. Non-secret settings come from `config.toml` (username, dirs, vision model, rate limit); `--config` overrides the path. - `uv run bggpipe <stage>` — run a pipeline stage. Non-secret settings come from `config.toml` (username, dirs, vision model, rate limit); `--config` overrides the path.
- `uv run pytest`105 tests, all offline against fixtures. Tests marked `live` hit the real BGG API (read-only) and are skipped unless you pass `--run-live`. - `uv run pytest`the suite runs fully offline against fixtures. Tests marked `live` hit the real BGG API (read-only) and are skipped unless you pass `--run-live`.
- `uv run ruff check` / `uv run ruff format` — lint (rules E, F, I, UP, B, SIM) and format. - `uv run ruff check` / `uv run ruff format` — lint (rules E, F, I, UP, B, SIM) and format.
## Layout ## Layout
+36
View File
@@ -0,0 +1,36 @@
"""Shared plumbing for the two stub-fixture generators.
Both write into the same cache dirs, so they must agree on XML escaping
(a title containing & or " must not produce malformed XML) and on the
provenance-marker text the upload guard depends on.
"""
from __future__ import annotations
from pathlib import Path
CACHE_MARKER_TEXT = (
"This cache contains hand-written stub XML, not real BGG "
"responses. Data resolved from it must not be uploaded.\n"
)
DATA_MARKER_TEXT = (
"The CSVs in this directory were resolved from hand-written stub "
"fixtures, not real BGG data — version_ids are SYNTHETIC. The "
"upload stage refuses to run while this file exists. Delete it "
"only after re-resolving against real recorded fixtures "
"(BGG_API_TOKEN + scripts/record_fixtures.py + resolve --force).\n"
)
def esc(text: str) -> str:
"""Minimal XML attribute/text escaping for hand-built fixture strings."""
return str(text).replace("&", "&amp;").replace("<", "&lt;").replace('"', "&quot;")
def write_cache_marker(target: Path) -> None:
(target / "STUB_FIXTURES.marker").write_text(CACHE_MARKER_TEXT)
def write_data_marker(data_dir: Path = Path("data")) -> None:
data_dir.mkdir(parents=True, exist_ok=True)
(data_dir / "STUB_DATA.marker").write_text(DATA_MARKER_TEXT)
+10 -21
View File
@@ -17,6 +17,8 @@ from __future__ import annotations
from pathlib import Path from pathlib import Path
from fixture_common import esc, write_cache_marker, write_data_marker
from bggpipe.bgg_client import cache_key from bggpipe.bgg_client import cache_key
TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache")) TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
@@ -295,7 +297,7 @@ VERSIONS: dict[int, list[tuple]] = {
def search_xml(results: list[tuple]) -> str: def search_xml(results: list[tuple]) -> str:
items = "".join( items = "".join(
f'<item type="{type_}" id="{bgg_id}">' f'<item type="{type_}" id="{bgg_id}">'
f'<name type="{name_type}" value="{_esc(name)}"/>' f'<name type="{name_type}" value="{esc(name)}"/>'
f'<yearpublished value="{year}"/></item>' f'<yearpublished value="{year}"/></item>'
for bgg_id, name, year, type_, name_type in results for bgg_id, name, year, type_, name_type in results
) )
@@ -306,12 +308,12 @@ def stats_xml(things: list[tuple]) -> str:
items = "" items = ""
for bgg_id, name, year, type_, owned, rank, publishers in things: for bgg_id, name, year, type_, owned, rank, publishers in things:
links = "".join( links = "".join(
f'<link type="boardgamepublisher" id="1" value="{_esc(p)}"/>' f'<link type="boardgamepublisher" id="1" value="{esc(p)}"/>'
for p in publishers for p in publishers
) )
items += ( items += (
f'<item type="{type_}" id="{bgg_id}">' f'<item type="{type_}" id="{bgg_id}">'
f'<name type="primary" value="{_esc(name)}"/>' f'<name type="primary" value="{esc(name)}"/>'
f'<yearpublished value="{year}"/>{links}' f'<yearpublished value="{year}"/>{links}'
f"<statistics><ratings>" f"<statistics><ratings>"
f'<owned value="{owned}"/>' f'<owned value="{owned}"/>'
@@ -326,14 +328,14 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
version_items = "" version_items = ""
for vid, name, year, publishers, languages in versions: for vid, name, year, publishers, languages in versions:
links = "".join( links = "".join(
f'<link type="boardgamepublisher" id="1" value="{_esc(p)}"/>' f'<link type="boardgamepublisher" id="1" value="{esc(p)}"/>'
for p in publishers for p in publishers
) + "".join( ) + "".join(
f'<link type="language" id="1" value="{_esc(lang)}"/>' for lang in languages f'<link type="language" id="1" value="{esc(lang)}"/>' for lang in languages
) )
version_items += ( version_items += (
f'<item type="boardgameversion" id="{vid}">' f'<item type="boardgameversion" id="{vid}">'
f'<name type="primary" value="{_esc(name)}"/>' f'<name type="primary" value="{esc(name)}"/>'
f'<yearpublished value="{year}"/>{links}</item>' f'<yearpublished value="{year}"/>{links}</item>'
) )
return ( return (
@@ -343,9 +345,6 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
) )
def _esc(text: str) -> str:
return text.replace("&", "&amp;").replace("<", "&lt;").replace('"', "&quot;")
def main() -> None: def main() -> None:
files: dict[str, str] = {} files: dict[str, str] = {}
@@ -364,18 +363,8 @@ def main() -> None:
(target / name).write_text(xml) (target / name).write_text(xml)
# provenance marker: anything resolved from this cache is stub-derived # provenance marker: anything resolved from this cache is stub-derived
# and NOT upload-ready; re-recording real fixtures removes the marker # and NOT upload-ready; re-recording real fixtures removes the marker
(target / "STUB_FIXTURES.marker").write_text( write_cache_marker(target)
"This cache contains hand-written stub XML, not real BGG " write_data_marker(DATA_MARKER.parent)
"responses. Data resolved from it must not be uploaded.\n"
)
DATA_MARKER.parent.mkdir(parents=True, exist_ok=True)
DATA_MARKER.write_text(
"The CSVs in this directory were resolved from hand-written stub "
"fixtures, not real BGG data — version_ids are SYNTHETIC. The "
"upload stage refuses to run while this file exists. Delete it "
"only after re-resolving against real recorded fixtures "
"(BGG_API_TOKEN + scripts/record_fixtures.py + resolve --force).\n"
)
print(f"Wrote {len(files)} fixture file(s) to {' and '.join(map(str, TARGETS))}") print(f"Wrote {len(files)} fixture file(s) to {' and '.join(map(str, TARGETS))}")
print(f"Wrote {DATA_MARKER} (committed; upload refuses while it exists)") print(f"Wrote {DATA_MARKER} (committed; upload refuses while it exists)")
+4 -5
View File
@@ -14,6 +14,8 @@ from __future__ import annotations
from pathlib import Path from pathlib import Path
from fixture_common import esc, write_cache_marker
from bggpipe.bgg_client import cache_key from bggpipe.bgg_client import cache_key
FIXTURE_CACHE = Path("tests/fixtures/bgg_cache") FIXTURE_CACHE = Path("tests/fixtures/bgg_cache")
@@ -24,7 +26,7 @@ def search_item(bgg_id: int, name: str, year: int | None, type_: str) -> str:
year_xml = f'<yearpublished value="{year}"/>' if year else "" year_xml = f'<yearpublished value="{year}"/>' if year else ""
return ( return (
f'<item type="{type_}" id="{bgg_id}">' f'<item type="{type_}" id="{bgg_id}">'
f'<name type="primary" value="{name}"/>{year_xml}</item>' f'<name type="primary" value="{esc(name)}"/>{year_xml}</item>'
) )
@@ -104,10 +106,7 @@ THINGS = {
def main() -> None: def main() -> None:
FIXTURE_CACHE.mkdir(parents=True, exist_ok=True) FIXTURE_CACHE.mkdir(parents=True, exist_ok=True)
(FIXTURE_CACHE / "STUB_FIXTURES.marker").write_text( write_cache_marker(FIXTURE_CACHE)
"This cache contains hand-written stub XML, not real BGG "
"responses. Data resolved from it must not be uploaded.\n"
)
for query, items in SEARCHES.items(): for query, items in SEARCHES.items():
key = cache_key("search", {"query": query, "type": SEARCH_TYPES}) key = cache_key("search", {"query": query, "type": SEARCH_TYPES})
total = items.count("<item ") total = items.count("<item ")
+17 -2
View File
@@ -18,7 +18,10 @@ from urllib.parse import urlencode
import httpx import httpx
from bggpipe import __version__
from bggpipe.config import Config
from bggpipe.models import ( from bggpipe.models import (
BGGResponseError,
CollectionItem, CollectionItem,
SearchResult, SearchResult,
ThingDetails, ThingDetails,
@@ -65,7 +68,7 @@ class BGGClient:
self._monotonic = monotonic self._monotonic = monotonic
self._rng = rng or random.Random() self._rng = rng or random.Random()
self._last_request: float | None = None self._last_request: float | None = None
headers = {"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"} headers = {"User-Agent": f"bggpipe/{__version__} (shelf-collection pipeline)"}
# BGG requires registered applications since 2025: the token from # BGG requires registered applications since 2025: the token from
# https://boardgamegeek.com/applications must accompany every request. # https://boardgamegeek.com/applications must accompany every request.
# Env var only — never config, disk, or logs. # Env var only — never config, disk, or logs.
@@ -117,6 +120,12 @@ class BGGClient:
continue continue
response.raise_for_status() response.raise_for_status()
if "<errors" in response.text[:120]:
# BGG serves some errors as HTTP 200 <errors> XML (bad
# username etc.) — caching one would poison every re-run
raise BGGResponseError(
f"BGG error document for /{endpoint}: {response.text[:200]}"
)
self.cache_dir.mkdir(parents=True, exist_ok=True) self.cache_dir.mkdir(parents=True, exist_ok=True)
cache_path.write_text(response.text) cache_path.write_text(response.text)
return response.text return response.text
@@ -137,6 +146,7 @@ class BGGClient:
def things( def things(
self, self,
ids: Iterable[int], ids: Iterable[int],
*,
stats: bool = False, stats: bool = False,
versions: bool = False, versions: bool = False,
) -> list[ThingDetails]: ) -> list[ThingDetails]:
@@ -155,9 +165,9 @@ class BGGClient:
def collection( def collection(
self, self,
username: str, username: str,
*,
subtype: str | None = None, subtype: str | None = None,
version: bool = True, version: bool = True,
*,
refresh: bool = False, refresh: bool = False,
) -> list[CollectionItem]: ) -> list[CollectionItem]:
params = {"username": username, "own": "1"} params = {"username": username, "own": "1"}
@@ -179,3 +189,8 @@ class BGGClient:
) )
seen = {item.coll_id for item in base} seen = {item.coll_id for item in base}
return base + [e for e in expansions if e.coll_id not in seen] return base + [e for e in expansions if e.coll_id not in seen]
def client_for(cfg: Config) -> BGGClient:
"""The standard injection fallback: every stage's `client or client_for(cfg)`."""
return BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
+8 -3
View File
@@ -7,7 +7,7 @@ from typing import Annotated
import typer import typer
from bggpipe.config import load_config from bggpipe.config import DEFAULT_REVIEW_PORT, load_config
app = typer.Typer( app = typer.Typer(
help="Shelf-to-BGG collection pipeline.", help="Shelf-to-BGG collection pipeline.",
@@ -56,7 +56,9 @@ def review(
web: Annotated[ web: Annotated[
bool, typer.Option("--web", help="Serve the review UI on localhost") bool, typer.Option("--web", help="Serve the review UI on localhost")
] = False, ] = False,
port: Annotated[int, typer.Option("--port", help="Port for --web")] = 8377, port: Annotated[
int, typer.Option("--port", help="Port for --web")
] = DEFAULT_REVIEW_PORT,
dev: Annotated[ dev: Annotated[
bool, typer.Option("--dev", help="With --web: restart on source changes") bool, typer.Option("--dev", help="With --web: restart on source changes")
] = False, ] = False,
@@ -122,7 +124,10 @@ def upload(
@app.command() @app.command()
def enrich( def enrich(
refresh: Annotated[bool, typer.Option("--refresh")] = False, refresh: Annotated[
bool,
typer.Option("--refresh", help="Re-fetch metadata (ranks/ratings drift)"),
] = False,
config: ConfigOpt = None, config: ConfigOpt = None,
) -> None: ) -> None:
"""Stage 6: fetch full game + version metadata into games.json.""" """Stage 6: fetch full game + version metadata into games.json."""
+41
View File
@@ -10,10 +10,12 @@ from __future__ import annotations
import os import os
import tomllib import tomllib
import warnings
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
from pathlib import Path from pathlib import Path
DEFAULT_CONFIG_PATH = Path("config.toml") DEFAULT_CONFIG_PATH = Path("config.toml")
DEFAULT_REVIEW_PORT = 8377
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -40,6 +42,38 @@ class Config:
def matches_path(self) -> Path: def matches_path(self) -> Path:
return self.data_dir / "matches.csv" return self.data_dir / "matches.csv"
@property
def extract_raw_dir(self) -> Path:
return self.data_dir / "extract_raw"
@property
def to_add_path(self) -> Path:
return self.data_dir / "to_add.csv"
@property
def to_update_path(self) -> Path:
return self.data_dir / "to_update.csv"
@property
def upload_log_path(self) -> Path:
return self.data_dir / "upload_log.csv"
@property
def games_path(self) -> Path:
return self.data_dir / "games.json"
@property
def dismissed_path(self) -> Path:
return self.data_dir / "unidentified_dismissed.json"
@property
def stub_marker_paths(self) -> tuple[Path, Path]:
# gitignored (travels with the stub XML) + committed (guards clones)
return (
self.cache_dir / "STUB_FIXTURES.marker",
self.data_dir / "STUB_DATA.marker",
)
def load_config(path: Path | None = None) -> Config: def load_config(path: Path | None = None) -> Config:
cfg = Config() cfg = Config()
@@ -53,6 +87,13 @@ def load_config(path: Path | None = None) -> Config:
"rate_limit_seconds": float, "rate_limit_seconds": float,
} }
updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw} updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw}
if unknown := sorted(raw.keys() - known.keys()):
# a typo'd key silently falling back to defaults is a debugging
# trap ("No photos found in photos/") — say so up front
warnings.warn(
f"{p}: ignoring unknown key(s): {', '.join(unknown)}",
stacklevel=2,
)
cfg = replace(cfg, **updates) cfg = replace(cfg, **updates)
if username := os.environ.get("BGG_USERNAME"): if username := os.environ.get("BGG_USERNAME"):
cfg = replace(cfg, bgg_username=username) cfg = replace(cfg, bgg_username=username)
+38 -26
View File
@@ -8,11 +8,12 @@ Two collection sources:
logged-in-user exemption. logged-in-user exemption.
Outputs both artifacts: Outputs both artifacts:
- to_add.csv — recognized games not in the collection (new entries); - to_add.csv — recognized games not in the collection, including
additional copies whose confident version matches no owned copy;
- to_update.csv — owned, VERSION-LESS entries where matching produced a - to_update.csv — owned, VERSION-LESS entries where matching produced a
confident version (version_auto/version_approved). Strictly additive: confident version (version_auto/version_approved). Strictly additive:
entries that already carry a version are never touched — a photo/version entries that already carry a version are never touched — a further copy
disagreement is reported in the summary instead. with a different version becomes a to_add row instead.
""" """
from __future__ import annotations from __future__ import annotations
@@ -24,9 +25,13 @@ from pathlib import Path
import typer import typer
from bggpipe.bgg_client import BGGClient from bggpipe.bgg_client import BGGClient, client_for
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.models import CollectionItem, parse_collection from bggpipe.models import (
CONFIDENT_VERSION_STATUSES,
CollectionItem,
parse_collection,
)
from bggpipe.resolve import read_matches from bggpipe.resolve import read_matches
TO_ADD_COLUMNS = [ TO_ADD_COLUMNS = [
@@ -42,7 +47,6 @@ TO_ADD_COLUMNS = [
TO_UPDATE_COLUMNS = ["collid", "bgg_id", "bgg_name", "version_id", "version_name"] TO_UPDATE_COLUMNS = ["collid", "bgg_id", "bgg_name", "version_id", "version_name"]
SNAPSHOT_FILES = ("collection_snapshot_base.xml", "collection_snapshot_expansions.xml") SNAPSHOT_FILES = ("collection_snapshot_base.xml", "collection_snapshot_expansions.xml")
_CONFIDENT_VERSION = ("version_auto", "version_approved")
@dataclass @dataclass
@@ -50,7 +54,7 @@ class DiffResult:
to_add: list[dict] = field(default_factory=list) to_add: list[dict] = field(default_factory=list)
to_update: list[dict] = field(default_factory=list) to_update: list[dict] = field(default_factory=list)
already_owned: list[str] = field(default_factory=list) # title_raw already_owned: list[str] = field(default_factory=list) # title_raw
disagreements: list[str] = field(default_factory=list) second_copies: list[str] = field(default_factory=list) # notes for adds
unseen: list[CollectionItem] = field(default_factory=list) unseen: list[CollectionItem] = field(default_factory=list)
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
rejected: int = 0 rejected: int = 0
@@ -113,14 +117,15 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
if copies: if copies:
seen_object_ids.add(bgg_id) seen_object_ids.add(bgg_id)
confident = row["version_status"] in _CONFIDENT_VERSION and row["version_id"] confident = (
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
)
version_id = int(row["version_id"]) if confident else None version_id = int(row["version_id"]) if confident else None
if not copies: def add_row(row: dict = row, confident: bool = confident) -> dict:
photos = {p for p in row["source_photos"].split(";") if p} photos = {p for p in row["source_photos"].split(";") if p}
photos |= merged_photos.get(row["title_raw"], set()) photos |= merged_photos.get(row["title_raw"], set())
result.to_add.append( return {
{
"bgg_id": row["bgg_id"], "bgg_id": row["bgg_id"],
"bgg_name": row["bgg_name"], "bgg_name": row["bgg_name"],
"year": row["year"], "year": row["year"],
@@ -130,7 +135,9 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
"title_raw": row["title_raw"], "title_raw": row["title_raw"],
"source_photos": ";".join(sorted(photos)), "source_photos": ";".join(sorted(photos)),
} }
)
if not copies:
result.to_add.append(add_row())
continue continue
if not confident: if not confident:
@@ -161,12 +168,15 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
} }
) )
else: else:
# every copy already carries a (different) version — never touch it # no remaining copy can take this version: every copy either
result.already_owned.append(row["title_raw"]) # already carries a different version or was consumed by another
result.disagreements.append( # match — per spec this is an additional physical copy to ADD
f"{row['title_raw']}: photo suggests version " # (existing entries are never touched)
f"{row['version_name']!r} ({row['version_id']}) but the " result.to_add.append(add_row())
"collection entry already has a version set — not changing it" result.second_copies.append(
f"{row['title_raw']}: adding as a NEW copy with version "
f"{row['version_name']!r} ({row['version_id']}) — every "
"existing entry of this game keeps its current version"
) )
result.unseen = [ result.unseen = [
@@ -177,10 +187,12 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None: def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None:
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", newline="") as f: tmp = path.with_name(path.name + ".tmp")
with tmp.open("w", newline="") as f:
writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore") writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore")
writer.writeheader() writer.writeheader()
writer.writerows(rows) writer.writerows(rows)
os.replace(tmp, path) # atomic: a killed diff never leaves a torn queue
def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult: def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
@@ -191,8 +203,8 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
if os.environ.get("BGG_API_TOKEN") and cfg.bgg_username: if os.environ.get("BGG_API_TOKEN") and cfg.bgg_username:
typer.echo("Fetching live collection from BGG…") typer.echo("Fetching live collection from BGG…")
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds) client = client or client_for(cfg)
collection = client.collection_full(cfg.bgg_username) collection = client.collection_full(cfg.bgg_username, refresh=True)
else: else:
typer.echo( typer.echo(
"No BGG_API_TOKEN — using collection snapshot files in " "No BGG_API_TOKEN — using collection snapshot files in "
@@ -202,8 +214,8 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
result = compute_diff(rows, collection) result = compute_diff(rows, collection)
_write_csv(cfg.data_dir / "to_add.csv", TO_ADD_COLUMNS, result.to_add) _write_csv(cfg.to_add_path, TO_ADD_COLUMNS, result.to_add)
_write_csv(cfg.data_dir / "to_update.csv", TO_UPDATE_COLUMNS, result.to_update) _write_csv(cfg.to_update_path, TO_UPDATE_COLUMNS, result.to_update)
merged_note = f" · {result.merged} merged duplicate(s)" if result.merged else "" merged_note = f" · {result.merged} merged duplicate(s)" if result.merged else ""
typer.echo( typer.echo(
@@ -212,9 +224,9 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
f"update(s) · {len(result.pending)} pending review · " f"update(s) · {len(result.pending)} pending review · "
f"{result.rejected} rejected{merged_note}" f"{result.rejected} rejected{merged_note}"
) )
if result.disagreements: if result.second_copies:
typer.echo("\nVersion disagreements (left untouched):") typer.echo("\nSecond copies (verify these on the dry run before upload):")
for line in result.disagreements: for line in result.second_copies:
typer.echo(f" - {line}") typer.echo(f" - {line}")
if result.pending: if result.pending:
typer.echo("\nStill pending review: " + ", ".join(result.pending)) typer.echo("\nStill pending review: " + ", ".join(result.pending))
+1 -1
View File
@@ -66,7 +66,7 @@ def run_enrich(
key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"] key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
targets.append((key, int(row["bgg_id"]), version)) targets.append((key, int(row["bgg_id"]), version))
games_path = cfg.data_dir / "games.json" games_path = cfg.games_path
games: dict = json.loads(games_path.read_text()) if games_path.exists() else {} games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games}) need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
+17 -7
View File
@@ -103,7 +103,7 @@ def prepare_image(path: Path) -> tuple[str, str]:
return base64.standard_b64encode(buffer.getvalue()).decode(), "image/jpeg" return base64.standard_b64encode(buffer.getvalue()).decode(), "image/jpeg"
def parse_vision_response(text: str) -> tuple[list[dict], list[dict]]: def parse_vision_response(text: str) -> tuple[list[dict], list[dict], int]:
"""Parse the model's JSON defensively: strip code fences, locate the """Parse the model's JSON defensively: strip code fences, locate the
payload amid any prose. Returns (title entries, unidentified sightings). payload amid any prose. Returns (title entries, unidentified sightings).
A bare JSON array (the pre-unidentified response shape) still parses — A bare JSON array (the pre-unidentified response shape) still parses —
@@ -133,7 +133,8 @@ def parse_vision_response(text: str) -> tuple[list[dict], list[dict]]:
if isinstance(u, dict) if isinstance(u, dict)
and (u.get("location") or u.get("partial_text") or u.get("art_notes")) and (u.get("location") or u.get("partial_text") or u.get("art_notes"))
] ]
return titles, unidentified dropped = len(titles_raw) - len(titles)
return titles, unidentified, dropped
def default_vision(model: str) -> VisionFn: def default_vision(model: str) -> VisionFn:
@@ -172,7 +173,9 @@ def extract_photo(photo: Path, vision: VisionFn) -> dict:
"""One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache """One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache
file format).""" file format)."""
image_b64, media_type = prepare_image(photo) image_b64, media_type = prepare_image(photo)
raw_entries, raw_unidentified = parse_vision_response(vision(image_b64, media_type)) raw_entries, raw_unidentified, dropped = parse_vision_response(
vision(image_b64, media_type)
)
unidentified = [ unidentified = [
{ {
"location": str(u.get("location") or "").strip(), "location": str(u.get("location") or "").strip(),
@@ -198,10 +201,10 @@ def extract_photo(photo: Path, vision: VisionFn) -> dict:
"source_photos": [photo.name], "source_photos": [photo.name],
} }
) )
return {"titles": entries, "unidentified": unidentified} return {"titles": entries, "unidentified": unidentified, "dropped": dropped}
def _cues_conflict(a: dict, b: dict) -> bool: def cues_conflict(a: dict, b: dict) -> bool:
"""Two sightings conflict if any edition cue is set on both and differs — """Two sightings conflict if any edition cue is set on both and differs —
that means visibly different boxes, i.e. separate editions (spec).""" that means visibly different boxes, i.e. separate editions (spec)."""
for key in ("publisher_hint", "edition_hint", "language_hint"): for key in ("publisher_hint", "edition_hint", "language_hint"):
@@ -238,7 +241,7 @@ def dedupe_entries(entries: list[dict]) -> list[dict]:
for existing in result: for existing in result:
if existing["title_normalized"] == entry[ if existing["title_normalized"] == entry[
"title_normalized" "title_normalized"
] and not _cues_conflict(existing, entry): ] and not cues_conflict(existing, entry):
existing.update(_merge(existing, entry)) existing.update(_merge(existing, entry))
break break
else: else:
@@ -298,7 +301,7 @@ def run_extract(
) )
raise typer.Exit(code=1) raise typer.Exit(code=1)
raw_dir = cfg.data_dir / "extract_raw" raw_dir = cfg.extract_raw_dir
raw_dir.mkdir(parents=True, exist_ok=True) raw_dir.mkdir(parents=True, exist_ok=True)
vision = vision or default_vision(cfg.model) vision = vision or default_vision(cfg.model)
@@ -314,6 +317,13 @@ def run_extract(
if result["unidentified"] if result["unidentified"]
else "" else ""
) )
if result.get("dropped"):
# the prompt forbids silently omitting a box; so do we
note += (
f" — DROPPED {result['dropped']} malformed entr"
f"{'y' if result['dropped'] == 1 else 'ies'}; inspect "
f"{raw_path}"
)
typer.echo(f" {photo.name}: {len(result['titles'])} title(s){note}") typer.echo(f" {photo.name}: {len(result['titles'])} title(s){note}")
deduped, unidentified = rebuild_artifacts( deduped, unidentified = rebuild_artifacts(
+20 -2
View File
@@ -12,6 +12,12 @@ class BGGResponseError(Exception):
"""The API returned a well-formed error document (e.g. bad username).""" """The API returned a well-formed error document (e.g. bad username)."""
# The one status predicate the whole pipeline shares: a version is trusted
# for diff/upload/enrich only when matching produced it confidently or a
# human approved it.
CONFIDENT_VERSION_STATUSES = ("version_auto", "version_approved")
@dataclass(frozen=True) @dataclass(frozen=True)
class SearchResult: class SearchResult:
bgg_id: int bgg_id: int
@@ -209,6 +215,16 @@ def parse_things_full(xml_text: str) -> list[dict]:
return games return games
def _required_attr(item, name: str) -> str:
value = item.get(name)
if not value:
raise BGGResponseError(
f"collection item missing {name!r} — truncated or unexpected "
"response; refusing to feed it to the diff"
)
return value
def parse_collection(xml_text: str) -> list[CollectionItem]: def parse_collection(xml_text: str) -> list[CollectionItem]:
items = [] items = []
for item in _root(xml_text).findall("item"): for item in _root(xml_text).findall("item"):
@@ -217,8 +233,10 @@ def parse_collection(xml_text: str) -> list[CollectionItem]:
version_item = item.find("version/item") version_item = item.find("version/item")
items.append( items.append(
CollectionItem( CollectionItem(
object_id=int(item.get("objectid", 0)), # strict: a missing id coerced to 0 would collide in the
coll_id=int(item.get("collid", 0)), # collid dedupe and silently drop owned games from the diff
object_id=int(_required_attr(item, "objectid")),
coll_id=int(_required_attr(item, "collid")),
name=item.findtext("name", default=""), name=item.findtext("name", default=""),
subtype=item.get("subtype", "boardgame"), subtype=item.get("subtype", "boardgame"),
own=status is not None and status.get("own") == "1", own=status is not None and status.get("own") == "1",
+70 -47
View File
@@ -12,16 +12,17 @@ import csv
import json import json
import os import os
import re import re
from collections import Counter
from dataclasses import dataclass, field from dataclasses import dataclass, field
from pathlib import Path from pathlib import Path
import typer import typer
from rapidfuzz import fuzz from rapidfuzz import fuzz
from bggpipe.bgg_client import BGGAuthError, BGGClient from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.extract import _cues_conflict from bggpipe.extract import cues_conflict
from bggpipe.models import GameVersion from bggpipe.models import CONFIDENT_VERSION_STATUSES, GameVersion
from bggpipe.normalize import normalize_title from bggpipe.normalize import normalize_title
FUZZY_THRESHOLD = 90 FUZZY_THRESHOLD = 90
@@ -30,6 +31,10 @@ FUZZY_THRESHOLD = 90
DOMINANCE_MIN_OWNED = 100 DOMINANCE_MIN_OWNED = 100
DOMINANCE_FACTOR = 10 DOMINANCE_FACTOR = 10
VERSION_PLAUSIBLE_SCORE = 2 VERSION_PLAUSIBLE_SCORE = 2
# "Is this the publisher on the box?" — one answer, asked in two places
# (candidate tie-break and version scoring): the sites must move together.
PUBLISHER_MATCH_THRESHOLD = 85
EDITION_NAME_THRESHOLD = 80
MATCH_COLUMNS = [ MATCH_COLUMNS = [
"title_raw", "title_raw",
@@ -45,6 +50,7 @@ MATCH_COLUMNS = [
"version_candidates_json", "version_candidates_json",
"source_photos", "source_photos",
"merged_into", "merged_into",
"dedupe_veto",
] ]
@@ -127,6 +133,8 @@ class MatchRow:
self.version_candidates, ensure_ascii=False self.version_candidates, ensure_ascii=False
), ),
"source_photos": ";".join(self.source_photos), "source_photos": ";".join(self.source_photos),
"merged_into": "",
"dedupe_veto": "",
} }
@@ -198,6 +206,10 @@ def _plausible_candidates(
by_id: dict[int, Candidate] = {} by_id: dict[int, Candidate] = {}
for result in client.search(query): for result in client.search(query):
norm = normalize_title(result.name) norm = normalize_title(result.name)
# a fully non-Latin title normalizes to "" — empty-vs-empty is not a
# match (and token_sort_ratio("", "") is 100, so guard fuzzy too)
if not entry.title_normalized:
continue
exact = norm == entry.title_normalized exact = norm == entry.title_normalized
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized) fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
if not exact and fuzzy < FUZZY_THRESHOLD: if not exact and fuzzy < FUZZY_THRESHOLD:
@@ -259,13 +271,19 @@ def _publisher_pick(entry: TitleEntry, top: list[Candidate]) -> Candidate | None
candidate is from that publisher, that's the game.""" candidate is from that publisher, that's the game."""
if not entry.publisher_hint: if not entry.publisher_hint:
return None return None
if len({c.type for c in top}) > 1:
# mixed base-game/expansion candidates are never auto-resolved —
# the same veto _dominant applies (spec's top failure mode); an
# alternate name can make an expansion "exact" too
return None
hint = normalize_title(entry.publisher_hint) hint = normalize_title(entry.publisher_hint)
matches = [ matches = [
c c
for c in top for c in top
if c.exact if c.exact
and any( and any(
fuzz.partial_ratio(hint, normalize_title(p)) >= 85 for p in c.publishers fuzz.partial_ratio(hint, normalize_title(p)) >= PUBLISHER_MATCH_THRESHOLD
for p in c.publishers
) )
] ]
return matches[0] if len(matches) == 1 else None return matches[0] if len(matches) == 1 else None
@@ -296,7 +314,7 @@ def _score_version(entry: TitleEntry, version: GameVersion) -> int:
if entry.publisher_hint: if entry.publisher_hint:
hint = normalize_title(entry.publisher_hint) hint = normalize_title(entry.publisher_hint)
if any( if any(
fuzz.partial_ratio(hint, normalize_title(p)) >= 85 fuzz.partial_ratio(hint, normalize_title(p)) >= PUBLISHER_MATCH_THRESHOLD
for p in version.publishers for p in version.publishers
): ):
score += 2 score += 2
@@ -312,18 +330,24 @@ def _score_version(entry: TitleEntry, version: GameVersion) -> int:
and fuzz.token_set_ratio( and fuzz.token_set_ratio(
normalize_title(entry.edition_hint), normalize_title(version.name) normalize_title(entry.edition_hint), normalize_title(version.name)
) )
>= 80 >= EDITION_NAME_THRESHOLD
): ):
score += 2 score += 2
return score return score
def _resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None: def resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None:
"""Fill version_* fields on an auto/approved row. Never guess (spec).""" """Fill version_* fields on an auto/approved row. Never guess (spec).
Public: review's manual-id path calls this too."""
if not entry.has_version_cues: if not entry.has_version_cues:
row.version_status = "version_unknown" row.version_status = "version_unknown"
return return
(thing,) = client.things([row.bgg_id], versions=True) things = client.things([row.bgg_id], versions=True)
if not things:
# a manually-typed id BGG doesn't know: nothing to offer
row.version_status = "version_unknown"
return
thing = things[0]
scored = sorted( scored = sorted(
((v, _score_version(entry, v)) for v in thing.versions), ((v, _score_version(entry, v)) for v in thing.versions),
key=lambda pair: -pair[1], key=lambda pair: -pair[1],
@@ -364,14 +388,10 @@ def resolve_entry(client: BGGClient, entry: TitleEntry) -> MatchRow:
break break
row = _classify(client, entry, cands) row = _classify(client, entry, cands)
if row.match_status == "auto": if row.match_status == "auto":
_resolve_version(client, entry, row) resolve_version(client, entry, row)
return row return row
def _row_key(title_raw: str, source_photos: str) -> tuple[str, str]:
return (title_raw, source_photos)
@dataclass(frozen=True) @dataclass(frozen=True)
class MergeEvent: class MergeEvent:
loser_title: str loser_title: str
@@ -409,9 +429,13 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
for row in rows: for row in rows:
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]: if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
continue continue
if row.get("dedupe_veto"):
# a human already ruled "this is a genuinely separate copy" —
# re-running resolve must never overturn that (spec: re-runs
# lose no work, least of all review decisions)
continue
confident = ( confident = (
row["version_status"] in ("version_auto", "version_approved") row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
and row["version_id"]
) )
key = (row["bgg_id"], row["version_id"] if confident else "") key = (row["bgg_id"], row["version_id"] if confident else "")
groups.setdefault(key, []).append(row) groups.setdefault(key, []).append(row)
@@ -429,7 +453,7 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
) )
survivor = group[0] survivor = group[0]
for loser in group[1:]: for loser in group[1:]:
if _cues_conflict(cues(survivor), cues(loser)): if cues_conflict(cues(survivor), cues(loser)):
continue # conflicting edition cues: genuinely two copies continue # conflicting edition cues: genuinely two copies
loser["match_status"] = "merged" loser["match_status"] = "merged"
loser["merged_into"] = survivor["title_raw"] loser["merged_into"] = survivor["title_raw"]
@@ -449,8 +473,9 @@ def read_matches(path: Path) -> list[dict[str, str]]:
return [] return []
with path.open(newline="") as f: with path.open(newline="") as f:
rows = list(csv.DictReader(f)) rows = list(csv.DictReader(f))
for row in rows: # files written before the merged_into column existed for row in rows: # files written before these columns existed
row.setdefault("merged_into", "") row.setdefault("merged_into", "")
row.setdefault("dedupe_veto", "")
return rows return rows
@@ -467,39 +492,40 @@ def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
os.replace(tmp, path) os.replace(tmp, path)
def read_existing_keys(path: Path) -> set[tuple[str, str]]:
if not path.exists():
return set()
with path.open(newline="") as f:
return {_row_key(r["title_raw"], r["source_photos"]) for r in csv.DictReader(f)}
def append_rows(path: Path, rows: list[MatchRow]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
is_new = not path.exists()
with path.open("a", newline="") as f:
writer = csv.DictWriter(f, fieldnames=MATCH_COLUMNS)
if is_new:
writer.writeheader()
for row in rows:
writer.writerow(row.to_csv())
def run_resolve( def run_resolve(
cfg: Config, *, force: bool = False, client: BGGClient | None = None cfg: Config, *, force: bool = False, client: BGGClient | None = None
) -> list[MatchRow]: ) -> list[MatchRow]:
entries = load_titles(cfg.titles_path) entries = load_titles(cfg.titles_path)
if force and cfg.matches_path.exists(): if force and cfg.matches_path.exists():
cfg.matches_path.unlink() cfg.matches_path.unlink()
existing = read_existing_keys(cfg.matches_path) existing_rows = read_matches(cfg.matches_path)
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds) client = client or client_for(cfg)
# Pair entries with existing rows BY TITLE, positionally, not by exact
# (title, photos) key: extract unions a new photo of an already-resolved
# game into its entry, and that must update the row's provenance — not
# re-resolve the game as a duplicate row. Same-title entries only stay
# separate when their cues conflict (two editions), and those pair up
# in stable file order on both sides.
rows_by_title: dict[str, list[dict]] = {}
for row in existing_rows:
rows_by_title.setdefault(row["title_raw"], []).append(row)
seen_per_title: dict[str, int] = {}
new_rows: list[MatchRow] = [] new_rows: list[MatchRow] = []
skipped = 0 skipped = 0
photos_updated = False
blocked: list[str] = [] blocked: list[str] = []
for entry in entries: for entry in entries:
key = _row_key(entry.title_raw, ";".join(entry.source_photos)) ix = seen_per_title.get(entry.title_raw, 0)
if key in existing: seen_per_title[entry.title_raw] = ix + 1
paired = rows_by_title.get(entry.title_raw, [])
if ix < len(paired):
row_dict = paired[ix]
photos = ";".join(entry.source_photos)
if row_dict["source_photos"] != photos:
row_dict["source_photos"] = photos
photos_updated = True
skipped += 1 skipped += 1
continue continue
try: try:
@@ -515,12 +541,11 @@ def run_resolve(
version = f" [{row.version_status}]" if row.version_status else "" version = f" [{row.version_status}]" if row.version_status else ""
typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}") typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}")
append_rows(cfg.matches_path, new_rows) all_rows = existing_rows + [row.to_csv() for row in new_rows]
all_rows = read_matches(cfg.matches_path)
merges = dedupe_matches(all_rows, entries) merges = dedupe_matches(all_rows, entries)
if new_rows or photos_updated or merges:
write_matches(cfg.matches_path, all_rows) # atomic full rewrite
if merges: if merges:
write_matches(cfg.matches_path, all_rows)
typer.echo("") typer.echo("")
for m in merges: for m in merges:
typer.echo( typer.echo(
@@ -528,9 +553,7 @@ def run_resolve(
f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong" f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong"
) )
counts: dict[str, int] = {} counts = Counter(row.match_status for row in new_rows)
for row in new_rows:
counts[row.match_status] = counts.get(row.match_status, 0) + 1
summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items())) summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items()))
typer.echo( typer.echo(
f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); " f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); "
+74 -29
View File
@@ -1,9 +1,11 @@
"""Stage 3 — human review of ambiguous/unmatched matches, then versions. """Stage 3 — the review decision engine, worn by two faces.
A plain rich prompt loop (deliberately dependency-light, per spec). Every ReviewSession owns all decision logic and every matches.csv write: the
decision rewrites matches.csv atomically, so quitting mid-review — q, decision API (decide_*/veto_merge/pending_rows/version_rows) serves both
Ctrl-C, or end of input — loses nothing; the next run resumes with this module's rich TUI prompt loop and webreview's FastAPI endpoints, and
whatever is still ambiguous/unmatched. The version pass is optional and reload_if_changed() lets either face follow external rewrites (a resolve
run in another terminal). Every decision rewrites matches.csv atomically,
so quitting mid-review loses nothing. The version pass is optional and
skippable: version review must never block getting games uploaded. skippable: version review must never block getting games uploaded.
""" """
@@ -14,18 +16,20 @@ from collections.abc import Callable
from pathlib import Path from pathlib import Path
import httpx import httpx
import typer
from rich.console import Console from rich.console import Console
from rich.panel import Panel from rich.panel import Panel
from rich.table import Table from rich.table import Table
from bggpipe.bgg_client import BGGAuthError, BGGClient, BGGQueueTimeout from bggpipe.bgg_client import BGGAuthError, BGGClient, BGGQueueTimeout, client_for
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.models import BGGResponseError from bggpipe.models import BGGResponseError
from bggpipe.resolve import ( from bggpipe.resolve import (
MatchRow, MatchRow,
_resolve_version, TitleEntry,
load_titles, load_titles,
read_matches, read_matches,
resolve_version,
write_matches, write_matches,
) )
@@ -57,8 +61,9 @@ class ReviewSession:
self.cfg = cfg self.cfg = cfg
self.console = console or Console() self.console = console or Console()
self.input_fn = input_fn or (lambda prompt: self.console.input(prompt)) self.input_fn = input_fn or (lambda prompt: self.console.input(prompt))
self.client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds) self.client = client or client_for(cfg)
self.decisions = 0 self.decisions = 0
self.warnings: list[str] = []
self._load() self._load()
# -- plumbing ------------------------------------------------------- # -- plumbing -------------------------------------------------------
@@ -99,8 +104,34 @@ class ReviewSession:
raise _Quit raise _Quit
return answer return answer
def _save(self) -> None: def _warn(self, message: str) -> None:
"""Degradations must be visible in BOTH faces: the console for the
TUI, and self.warnings for the web UI (whose console is a StringIO)."""
self.warnings.append(message)
self.console.print(f"[yellow]{message}[/yellow]")
def _save(self, row: dict | None = None) -> None:
"""Atomic write of the in-memory rows. If another process rewrote the
file since we loaded (resolve in a second terminal), reload first and
re-apply `row` — the one decision being saved — by identity, so
neither side's work is silently lost."""
if self.reload_if_changed() and row is not None:
key = (row["title_raw"], row["source_photos"])
for i, fresh in enumerate(self.rows):
if (fresh["title_raw"], fresh["source_photos"]) == key:
self.rows[i] = row
break
else:
self._warn(
f"{row['title_raw']!r} disappeared from matches.csv while "
"you decided — decision NOT saved"
)
return
try:
write_matches(self.cfg.matches_path, self.rows) write_matches(self.cfg.matches_path, self.rows)
except OSError:
self._load() # memory must never claim what disk doesn't hold
raise
self._loaded_mtimes = self._data_mtimes() # own writes aren't "changes" self._loaded_mtimes = self._data_mtimes() # own writes aren't "changes"
self.decisions += 1 self.decisions += 1
@@ -111,7 +142,7 @@ class ReviewSession:
row["year"] = str(candidate.get("year") or "") row["year"] = str(candidate.get("year") or "")
row["type"] = candidate.get("type") or "boardgame" row["type"] = candidate.get("type") or "boardgame"
self._fill_version(row) self._fill_version(row)
self._save() self._save(row)
def _fill_version(self, row: dict) -> None: def _fill_version(self, row: dict) -> None:
"""Try version resolution for a just-approved row. Degrades gracefully: """Try version resolution for a just-approved row. Degrades gracefully:
@@ -122,9 +153,9 @@ class ReviewSession:
return return
shim = MatchRow(title_raw=row["title_raw"], bgg_id=int(row["bgg_id"])) shim = MatchRow(title_raw=row["title_raw"], bgg_id=int(row["bgg_id"]))
try: try:
_resolve_version(self.client, entry, shim) resolve_version(self.client, entry, shim)
except _BGG_ERRORS as err: except _BGG_ERRORS as err:
self.console.print(f"[yellow]version lookup unavailable: {err}[/yellow]") self._warn(f"version lookup unavailable ({err}) — recorded version_unknown")
row["version_status"] = "version_unknown" row["version_status"] = "version_unknown"
return return
row["version_status"] = shim.version_status row["version_status"] = shim.version_status
@@ -155,9 +186,21 @@ class ReviewSession:
row as a distinct, human-confirmed match.""" row as a distinct, human-confirmed match."""
row["match_status"] = "approved" row["match_status"] = "approved"
row["merged_into"] = "" row["merged_into"] = ""
self._save() row["dedupe_veto"] = "1" # persists: resolve re-runs must not re-merge
self._save(row)
def cues_for(self, title_raw: str): def cues_for(
self, title_raw: str, source_photos: str | None = None
) -> TitleEntry | None:
"""The extraction entry behind a row. Same-title entries (two
editions of one game) are told apart by their photo set when given."""
if source_photos is not None:
for entry in self.titles:
if (
entry.title_raw == title_raw
and ";".join(entry.source_photos) == source_photos
):
return entry
return self._titles.get(title_raw) return self._titles.get(title_raw)
def decide_pick(self, row: dict, candidate: dict) -> None: def decide_pick(self, row: dict, candidate: dict) -> None:
@@ -168,7 +211,7 @@ class ReviewSession:
def decide_reject(self, row: dict) -> None: def decide_reject(self, row: dict) -> None:
row["match_status"] = "rejected" row["match_status"] = "rejected"
self._save() self._save(row)
def decide_version(self, row: dict, version_id: int | None) -> None: def decide_version(self, row: dict, version_id: int | None) -> None:
"""Pick a version from the row's stored candidates, or None -> unknown.""" """Pick a version from the row's stored candidates, or None -> unknown."""
@@ -186,7 +229,7 @@ class ReviewSession:
row["version_status"] = "version_approved" row["version_status"] = "version_approved"
row["version_id"] = str(version_id) row["version_id"] = str(version_id)
row["version_name"] = chosen.get("name") or "" row["version_name"] = chosen.get("name") or ""
self._save() self._save(row)
# -- displays ------------------------------------------------------- # -- displays -------------------------------------------------------
@@ -247,12 +290,19 @@ class ReviewSession:
def _manual_id(self, row: dict, bgg_id: int) -> None: def _manual_id(self, row: dict, bgg_id: int) -> None:
candidate: dict = {"bgg_id": bgg_id} candidate: dict = {"bgg_id": bgg_id}
try: try:
(thing,) = self.client.things([bgg_id]) things = self.client.things([bgg_id])
candidate.update(name=thing.name, year=thing.year, type=thing.type)
except _BGG_ERRORS as err: except _BGG_ERRORS as err:
self.console.print( self._warn(
f"[yellow]couldn't look up id {bgg_id} ({err}) — " f"couldn't look up id {bgg_id} ({err}) — recording the id with no name"
"recording the id with no name[/yellow]" )
else:
if things:
thing = things[0]
candidate.update(name=thing.name, year=thing.year, type=thing.type)
else:
self._warn(
f"BGG knows no game with id {bgg_id} — recording it "
"with no name (typo?)"
) )
self._apply_choice(row, candidate) self._apply_choice(row, candidate)
@@ -302,15 +352,10 @@ class ReviewSession:
# -- entry point ---------------------------------------------------- # -- entry point ----------------------------------------------------
def run(self) -> None: def run(self) -> None:
pending = [
r for r in self.rows if r["match_status"] in ("ambiguous", "unmatched")
]
try: try:
for row in pending: for row in self.pending_rows():
self._review_match_row(row) self._review_match_row(row)
versions = [ versions = self.version_rows()
r for r in self.rows if r["version_status"] == "version_ambiguous"
]
if versions: if versions:
answer = self._ask( answer = self._ask(
f"{len(versions)} game(s) have ambiguous editions. " f"{len(versions)} game(s) have ambiguous editions. "
@@ -343,6 +388,6 @@ def run_review(
session.console.print( session.console.print(
f"{cfg.matches_path} is empty — run `bggpipe resolve` first." f"{cfg.matches_path} is empty — run `bggpipe resolve` first."
) )
return session raise typer.Exit(code=1)
session.run() session.run()
return session return session
+48 -12
View File
@@ -65,6 +65,13 @@
} }
header kbd { background: rgba(255,255,255,.14); color: var(--paper); border-color: rgba(255,255,255,.25); } header kbd { background: rgba(255,255,255,.14); color: var(--paper); border-color: rgba(255,255,255,.25); }
main { max-width: 62rem; margin: 0 auto; padding: 1.4rem 1.2rem 6rem; } main { max-width: 62rem; margin: 0 auto; padding: 1.4rem 1.2rem 6rem; }
#banner { max-width: 62rem; margin: 0 auto; padding: 0 1.2rem; }
.banner {
border-radius: 6px; padding: .6rem .9rem; margin-top: .9rem;
font-size: .85rem; line-height: 1.4;
}
.banner.error { background: #f6dcd6; border: 1px solid var(--reject); color: #6b2417; }
.banner.warn { background: var(--kraft); border: 1px solid var(--brass-deep); color: #6b5b33; }
h2 { h2 {
color: var(--paper); color: var(--paper);
font-family: "Iowan Old Style", Palatino, Georgia, serif; font-family: "Iowan Old Style", Palatino, Georgia, serif;
@@ -236,6 +243,7 @@
<kbd>v</kbd> veto merge · <kbd>d</kbd> dismiss <kbd>v</kbd> veto merge · <kbd>d</kbd> dismiss
</span> </span>
</header> </header>
<div id="banner"></div>
<main id="main"></main> <main id="main"></main>
<script> <script>
"use strict"; "use strict";
@@ -245,19 +253,36 @@ let active = 0;
const esc = s => String(s ?? "").replace(/[&<>"']/g, const esc = s => String(s ?? "").replace(/[&<>"']/g,
c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c])); c => ({"&":"&amp;","<":"&lt;",">":"&gt;",'"':"&quot;","'":"&#39;"}[c]));
function showBanner(html) {
document.getElementById("banner").innerHTML = html;
}
function errorBanner(detail) {
showBanner(`<div class="banner error">couldn't load review state: ${esc(detail)}` +
` — is the server running? (check its terminal)</div>`);
}
async function refresh() { async function refresh() {
STATE = await (await fetch("/api/state")).json(); const res = await fetch("/api/state");
if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => res.statusText)}`);
STATE = await res.json();
render(); render();
} }
async function post(url, body) { async function post(url, body) {
const res = await fetch(url, { let res;
try {
res = await fetch(url, {
method: "POST", method: "POST",
headers: {"Content-Type": "application/json"}, headers: {"Content-Type": "application/json"},
body: JSON.stringify(body), body: JSON.stringify(body),
}); });
} catch (err) {
alert("That didn't save (no response from the server): " + err);
return;
}
if (!res.ok) { if (!res.ok) {
const detail = (await res.json()).detail || res.statusText; const detail = await res.json().then(d => d.detail).catch(() => res.statusText);
alert("That didn't save: " + detail); alert("That didn't save: " + detail);
return; return;
} }
@@ -298,7 +323,7 @@ function matchCard(row, idx) {
· rank ${esc(c.rank ?? "—")} · owned ${esc(c.owned ?? "—")}</span></span> · rank ${esc(c.rank ?? "—")} · owned ${esc(c.owned ?? "—")}</span></span>
</li>`).join(""); </li>`).join("");
return ` return `
<section class="card actionable" data-kind="match" data-idx="${idx}" <section class="card actionable" data-kind="match" data-rowix="${row.row_ix}"
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}"> data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
${shots(row.photos)} ${shots(row.photos)}
<div class="body"> <div class="body">
@@ -324,7 +349,7 @@ function versionCard(row, idx) {
· ${esc((v.languages || []).join(", "))} · score ${esc(v.score ?? "—")}</span></span> · ${esc((v.languages || []).join(", "))} · score ${esc(v.score ?? "—")}</span></span>
</li>`).join(""); </li>`).join("");
return ` return `
<section class="card actionable" data-kind="version" data-idx="${idx}" <section class="card actionable" data-kind="version" data-rowix="${row.row_ix}"
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}"> data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
<div class="body"> <div class="body">
<p class="title">${esc(row.bgg_name || row.title_raw)}</p> <p class="title">${esc(row.bgg_name || row.title_raw)}</p>
@@ -361,6 +386,8 @@ function ticket(s) {
function render() { function render() {
const m = document.getElementById("main"); const m = document.getElementById("main");
const s = STATE; const s = STATE;
showBanner((s.warnings || []).length
? `<div class="banner warn">${s.warnings.map(esc).join("<br>")}</div>` : "");
document.getElementById("tally").innerHTML = document.getElementById("tally").innerHTML =
`<span><b>${s.pending.length}</b> matches</span> `<span><b>${s.pending.length}</b> matches</span>
<span><b>${s.versions.length}</b> editions</span> <span><b>${s.versions.length}</b> editions</span>
@@ -480,13 +507,14 @@ function highlight() {
if (el) el.scrollIntoView({block: "nearest", behavior: "auto"}); if (el) el.scrollIntoView({block: "nearest", behavior: "auto"});
} }
const rowIx = card => card.dataset.rowix === undefined ? null : Number(card.dataset.rowix);
const decide = (card, action, bgg_id = null) => post("/api/decision", { const decide = (card, action, bgg_id = null) => post("/api/decision", {
title_raw: card.dataset.title, source_photos: card.dataset.photos, title_raw: card.dataset.title, source_photos: card.dataset.photos,
action, bgg_id, row_ix: rowIx(card), action, bgg_id,
}); });
const version = (card, action, version_id = null) => post("/api/version", { const version = (card, action, version_id = null) => post("/api/version", {
title_raw: card.dataset.title, source_photos: card.dataset.photos, title_raw: card.dataset.title, source_photos: card.dataset.photos,
action, version_id, row_ix: rowIx(card), action, version_id,
}); });
const dismiss = t => post("/api/dismiss", { const dismiss = t => post("/api/dismiss", {
photo: t.dataset.photo, location: t.dataset.location, photo: t.dataset.photo, location: t.dataset.location,
@@ -519,23 +547,31 @@ document.addEventListener("keydown", e => {
else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); } else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); }
}); });
refresh(); refresh().catch(err => errorBanner(err.message || err));
// Live-follow the data files: extract/resolve runs in another terminal show // Live-follow the data files: extract/resolve runs in another terminal show
// up on the next poll. Only re-render on an actual change (keeps the // up on the next poll. Only re-render on an actual change (keeps the
// keyboard cursor stable) and never mid-typing in a manual-ID input. // keyboard cursor stable) and never mid-typing in a manual-ID input.
let pollMisses = 0;
setInterval(async () => { setInterval(async () => {
const el = document.activeElement; const el = document.activeElement;
if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA")) return; if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA")) return;
let fresh = null;
try { try {
const fresh = await (await fetch("/api/state")).json(); const res = await fetch("/api/state");
if (!res.ok) throw new Error(`${res.status}`);
fresh = await res.json();
} catch (err) {
// transient during --dev restarts; persistent means the server is gone
if (++pollMisses >= 3) errorBanner(`lost contact (${err.message || err})`);
return;
}
if (pollMisses >= 3) showBanner(""); // recovered: clear the lost-contact banner
pollMisses = 0;
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) { if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
STATE = fresh; STATE = fresh;
render(); render();
} }
} catch {
/* server restarting (--dev) — retry on the next tick */
}
}, 3000); }, 3000);
</script> </script>
</body> </body>
+82 -25
View File
@@ -27,6 +27,7 @@ import os
import random import random
import re import re
import time import time
from collections import Counter
from collections.abc import Callable from collections.abc import Callable
from dataclasses import dataclass from dataclasses import dataclass
from datetime import UTC, datetime from datetime import UTC, datetime
@@ -35,7 +36,7 @@ from typing import Protocol
import typer import typer
from bggpipe.bgg_client import BGGAuthError, BGGClient from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.models import CollectionItem from bggpipe.models import CollectionItem
@@ -51,7 +52,7 @@ UPLOAD_LOG_COLUMNS = [
"timestamp", "timestamp",
"error", "error",
] ]
DONE_STATUSES = {"added", "updated", "already_present"} DONE_STATUSES = {"added", "added_no_version", "updated", "already_present"}
MAX_VERSION_PAGES = 40 MAX_VERSION_PAGES = 40
@@ -73,7 +74,7 @@ class UploadJob:
return ("add", self.bgg_id, self.version_id) return ("add", self.bgg_id, self.version_id)
def _row_key(row: dict) -> tuple[str, str, str]: def _job_key(row: dict) -> tuple[str, str, str]:
if row["action"] == "update": if row["action"] == "update":
return ("update", row["collid"], "") return ("update", row["collid"], "")
return ("add", row["bgg_id"], row["version_id"]) return ("add", row["bgg_id"], row["version_id"])
@@ -109,7 +110,7 @@ def build_queue(
done. Returns (jobs, skipped_done, skipped_failed).""" done. Returns (jobs, skipped_done, skipped_failed)."""
latest: dict[tuple[str, str, str], str] = {} latest: dict[tuple[str, str, str], str] = {}
for row in log_rows: for row in log_rows:
latest[_row_key(row)] = row["status"] latest[_job_key(row)] = row["status"]
candidates = [ candidates = [
UploadJob( UploadJob(
@@ -134,15 +135,25 @@ def build_queue(
jobs: list[UploadJob] = [] jobs: list[UploadJob] = []
skipped_done = skipped_failed = 0 skipped_done = skipped_failed = 0
deferred: list[UploadJob] = []
update_game_seen: set[str] = set()
for job in candidates: for job in candidates:
status = latest.get(job.key) status = latest.get(job.key)
if status in DONE_STATUSES: if status in DONE_STATUSES:
skipped_done += 1 skipped_done += 1
elif status == "failed" and not retry_failed: elif status == "failed" and not retry_failed:
skipped_failed += 1 skipped_failed += 1
elif job.action == "update" and job.bgg_id in update_game_seen:
# The row-edit flow finds rows by game name, not collid — a
# second same-game update this run could reopen the copy the
# first one just versioned and overwrite it. One per run; the
# next run (after --verify) picks up the rest.
deferred.append(job)
else: else:
if job.action == "update":
update_game_seen.add(job.bgg_id)
jobs.append(job) jobs.append(job)
return jobs, skipped_done, skipped_failed return jobs, skipped_done, skipped_failed, deferred
def _scrub(text: str) -> str: def _scrub(text: str) -> str:
@@ -155,6 +166,12 @@ def _scrub(text: str) -> str:
return text return text
class LoginError(RuntimeError):
"""Authentication is broken (bad credentials, Cloudflare block, changed
login page). Systemic by definition: retrying per-game would hammer the
login endpoint and poison upload_log.csv with misleading failures."""
class Uploader(Protocol): class Uploader(Protocol):
def add_game(self, job: UploadJob) -> tuple[str, str]: ... def add_game(self, job: UploadJob) -> tuple[str, str]: ...
@@ -215,11 +232,16 @@ class PlaywrightUploader:
return return
page = self._page page = self._page
self._goto(f"{BGG}/") self._goto(f"{BGG}/")
if "just a moment" in (page.title() or "").casefold():
raise LoginError(
"Cloudflare is challenging this browser ('Just a moment...') "
"— run headed (drop --headless) and click the widget once"
)
if self._signed_out(): if self._signed_out():
user = os.environ.get("BGG_USERNAME", "") user = os.environ.get("BGG_USERNAME", "")
password = os.environ.get("BGG_PASSWORD", "") password = os.environ.get("BGG_PASSWORD", "")
if not (user and password): if not (user and password):
raise RuntimeError( raise LoginError(
"BGG_USERNAME and BGG_PASSWORD env vars are required to log in" "BGG_USERNAME and BGG_PASSWORD env vars are required to log in"
) )
self._goto(f"{BGG}/login") self._goto(f"{BGG}/login")
@@ -229,7 +251,13 @@ class PlaywrightUploader:
page.locator("#inputUsername").fill(user) page.locator("#inputUsername").fill(user)
page.locator("#inputPassword").fill(password) page.locator("#inputPassword").fill(password)
page.get_by_role("button", name="Sign In").click() page.get_by_role("button", name="Sign In").click()
try:
page.wait_for_url(lambda url: "/login" not in url, timeout=120_000) page.wait_for_url(lambda url: "/login" not in url, timeout=120_000)
except self._timeout_error as err:
raise LoginError(
"login did not complete (wrong credentials, or the login "
"page changed — see docs/bgg-upload-flow.md)"
) from err
self._context.storage_state(path=str(self._storage_state)) self._context.storage_state(path=str(self._storage_state))
self._authed = True self._authed = True
@@ -266,6 +294,7 @@ class PlaywrightUploader:
if nxt.count() == 0 or nxt.is_disabled(): if nxt.count() == 0 or nxt.is_disabled():
break break
nxt.click() nxt.click()
self._page.wait_for_timeout(2_000) # etiquette: paginating hits BGG too
# Two-level dismissal: the sub-view has its own Cancel distinct from # Two-level dismissal: the sub-view has its own Cancel distinct from
# the main dialog's. # the main dialog's.
dialog.get_by_role("button", name="Cancel").first.click() dialog.get_by_role("button", name="Cancel").first.click()
@@ -283,13 +312,14 @@ class PlaywrightUploader:
"heading", name=re.compile(re.escape(job.name), re.I) "heading", name=re.compile(re.escape(job.name), re.I)
).wait_for(timeout=15_000) ).wait_for(timeout=15_000)
dialog.get_by_label("Own").check() dialog.get_by_label("Own").check()
note = "" status, note = "added", ""
if job.version_name and not self._select_version(dialog, job.version_name): if job.version_name and not self._select_version(dialog, job.version_name):
status = "added_no_version"
note = f"version {job.version_name!r} not in picker; added without version" note = f"version {job.version_name!r} not in picker; added without version"
dialog.get_by_role("button", name="Save").click() dialog.get_by_role("button", name="Save").click()
# The dialog is hidden after save, not removed from the DOM. # The dialog is hidden after save, not removed from the DOM.
dialog.wait_for(state="hidden", timeout=15_000) dialog.wait_for(state="hidden", timeout=15_000)
return "added", note return status, note
def update_entry(self, job: UploadJob) -> tuple[str, str]: def update_entry(self, job: UploadJob) -> tuple[str, str]:
"""Set the version on an EXISTING entry — strictly additive. """Set the version on an EXISTING entry — strictly additive.
@@ -332,6 +362,7 @@ def _process(
now: Callable[[], str], now: Callable[[], str],
) -> list[dict]: ) -> list[dict]:
results = [] results = []
consecutive: tuple[str, int] = ("", 0)
for i, job in enumerate(jobs): for i, job in enumerate(jobs):
if i: if i:
sleep(rng.uniform(2.0, 4.0)) # polite pacing between games (spec) sleep(rng.uniform(2.0, 4.0)) # polite pacing between games (spec)
@@ -340,6 +371,12 @@ def _process(
status, note = uploader.add_game(job) status, note = uploader.add_game(job)
else: else:
status, note = uploader.update_entry(job) status, note = uploader.update_entry(job)
except LoginError as exc:
# Systemic: every remaining job would fail identically. Abort
# WITHOUT logging failures, so the next run just retries.
typer.echo(f" aborting — login is broken: {_scrub(str(exc))}")
typer.echo(f" {len(jobs) - i} job(s) left untouched for the next run.")
break
except Exception as exc: # per-game isolation: log it, keep going except Exception as exc: # per-game isolation: log it, keep going
status, note = "failed", _scrub(f"{type(exc).__name__}: {exc}") status, note = "failed", _scrub(f"{type(exc).__name__}: {exc}")
row = { row = {
@@ -350,12 +387,25 @@ def _process(
"version_id": job.version_id, "version_id": job.version_id,
"status": status, "status": status,
"timestamp": now(), "timestamp": now(),
"error": note if status == "failed" else note, # the column carries degradation notes on successes too
# (e.g. added_no_version), not just failure text
"error": note,
} }
append_log_row(log_path, row) append_log_row(log_path, row)
results.append(row) results.append(row)
suffix = f"{note}" if note else "" suffix = f"{note}" if note else ""
typer.echo(f" {job.name}: {status}{suffix}") typer.echo(f" {job.name}: {status}{suffix}")
if status == "failed":
consecutive = (note, consecutive[1] + 1 if note == consecutive[0] else 1)
if consecutive[1] >= 3:
typer.echo(
" aborting — 3 identical consecutive failures look "
"systemic, not per-game; remaining jobs left for the "
"next run"
)
break
else:
consecutive = ("", 0)
return results return results
@@ -370,14 +420,18 @@ def verify_uploads(log_rows: list[dict], collection: list[CollectionItem]) -> li
problems = [] problems = []
latest: dict[tuple[str, str, str], dict] = {} latest: dict[tuple[str, str, str], dict] = {}
for row in log_rows: for row in log_rows:
latest[_row_key(row)] = row latest[_job_key(row)] = row
for row in latest.values(): for row in latest.values():
if row["status"] == "added": if row["status"] in ("added", "added_no_version"):
copies = by_object.get(int(row["bgg_id"]), []) copies = by_object.get(int(row["bgg_id"]), [])
if not copies: if not copies:
problems.append(f"{row['name']}: logged added but not in collection") problems.append(f"{row['name']}: logged added but not in collection")
elif row["version_id"] and not any( elif (
row["status"] == "added" # no_version: absence is expected
and row["version_id"]
and not any(
str(c.version_id or "") == row["version_id"] for c in copies str(c.version_id or "") == row["version_id"] for c in copies
)
): ):
problems.append( problems.append(
f"{row['name']}: in collection but no copy has " f"{row['name']}: in collection but no copy has "
@@ -419,11 +473,7 @@ def run_upload(
# cache marker travels with the stub XML (gitignored, so a fresh clone # cache marker travels with the stub XML (gitignored, so a fresh clone
# loses it), while data/STUB_DATA.marker is COMMITTED alongside the # loses it), while data/STUB_DATA.marker is COMMITTED alongside the
# stub-derived CSVs — so a clone can never upload placeholder ids. # stub-derived CSVs — so a clone can never upload placeholder ids.
markers = [ marker = next((m for m in cfg.stub_marker_paths if m.exists()), None)
cfg.cache_dir / "STUB_FIXTURES.marker",
cfg.data_dir / "STUB_DATA.marker",
]
marker = next((m for m in markers if m.exists()), None)
if marker is not None: if marker is not None:
if not dry_run: if not dry_run:
typer.echo( typer.echo(
@@ -439,12 +489,15 @@ def run_upload(
"placeholders, not real BGG data.\n" "placeholders, not real BGG data.\n"
) )
log_path = cfg.data_dir / "upload_log.csv" log_path = cfg.upload_log_path
to_add = _read_csv(cfg.data_dir / "to_add.csv") if not cfg.to_add_path.exists():
to_update = _read_csv(cfg.data_dir / "to_update.csv") typer.echo(f"{cfg.to_add_path} not found — run `bggpipe diff` first.")
raise typer.Exit(code=1)
to_add = _read_csv(cfg.to_add_path)
to_update = _read_csv(cfg.to_update_path)
log_rows = _read_csv(log_path) log_rows = _read_csv(log_path)
jobs, skipped_done, skipped_failed = build_queue( jobs, skipped_done, skipped_failed, deferred = build_queue(
to_add, to_update, log_rows, retry_failed=retry_failed to_add, to_update, log_rows, retry_failed=retry_failed
) )
if limit is not None: if limit is not None:
@@ -456,6 +509,12 @@ def run_upload(
f" (skipping {skipped_done} already done, {skipped_failed} " f" (skipping {skipped_done} already done, {skipped_failed} "
"previously failed — use --retry-failed)" "previously failed — use --retry-failed)"
) )
if deferred:
typer.echo(
f"{len(deferred)} version update(s) deferred: one update per game "
"per run (the row-edit flow can't target a collid) — re-run "
"upload after --verify confirms this batch."
)
results: list[dict] = [] results: list[dict] = []
if dry_run: if dry_run:
@@ -484,9 +543,7 @@ def run_upload(
results = _process(real, jobs, log_path, sleep=sleep, rng=rng, now=now) results = _process(real, jobs, log_path, sleep=sleep, rng=rng, now=now)
else: else:
results = _process(uploader, jobs, log_path, sleep=sleep, rng=rng, now=now) results = _process(uploader, jobs, log_path, sleep=sleep, rng=rng, now=now)
counts: dict[str, int] = {} counts = Counter(row["status"] for row in results)
for row in results:
counts[row["status"]] = counts.get(row["status"], 0) + 1
summary = " · ".join(f"{n} {status}" for status, n in sorted(counts.items())) summary = " · ".join(f"{n} {status}" for status, n in sorted(counts.items()))
typer.echo(f"\n{summary or 'nothing to do'}") typer.echo(f"\n{summary or 'nothing to do'}")
else: else:
@@ -501,7 +558,7 @@ def _run_verify(cfg: Config, client: BGGClient | None, log_path: Path) -> None:
if not cfg.bgg_username: if not cfg.bgg_username:
typer.echo("--verify needs BGG_USERNAME in the environment.") typer.echo("--verify needs BGG_USERNAME in the environment.")
return return
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds) client = client or client_for(cfg)
try: try:
collection = client.collection_full(cfg.bgg_username, refresh=True) collection = client.collection_full(cfg.bgg_username, refresh=True)
except BGGAuthError as exc: except BGGAuthError as exc:
+45 -15
View File
@@ -15,6 +15,8 @@ from __future__ import annotations
import io import io
import json import json
import os import os
import threading
from collections import Counter
from importlib import resources from importlib import resources
from pathlib import Path from pathlib import Path
@@ -26,11 +28,10 @@ from pydantic import BaseModel
from rich.console import Console from rich.console import Console
from bggpipe.bgg_client import BGGClient from bggpipe.bgg_client import BGGClient
from bggpipe.config import DEFAULT_REVIEW_PORT as DEFAULT_PORT # single home
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.review import ReviewSession from bggpipe.review import ReviewSession
DEFAULT_PORT = 8377
def load_thumbnails(cache_dir: Path) -> dict[int, str]: def load_thumbnails(cache_dir: Path) -> dict[int, str]:
"""bgg_id -> thumbnail URL, from cached /thing XML only (no live calls). """bgg_id -> thumbnail URL, from cached /thing XML only (no live calls).
@@ -41,7 +42,7 @@ def load_thumbnails(cache_dir: Path) -> dict[int, str]:
for path in cache_dir.glob("thing_*.xml"): for path in cache_dir.glob("thing_*.xml"):
try: try:
root = _safe_fromstring(path.read_text()) root = _safe_fromstring(path.read_text())
except Exception: # noqa: BLE001 — a corrupt cache file must not kill the UI except Exception: # a corrupt cache file must not kill the UI
continue continue
for item in root.findall("item"): for item in root.findall("item"):
thumb = (item.findtext("thumbnail") or "").strip() thumb = (item.findtext("thumbnail") or "").strip()
@@ -80,6 +81,7 @@ class DismissStore:
class DecisionBody(BaseModel): class DecisionBody(BaseModel):
title_raw: str title_raw: str
source_photos: str source_photos: str
row_ix: int | None = None
action: str # "pick" | "manual" | "reject" action: str # "pick" | "manual" | "reject"
bgg_id: int | None = None bgg_id: int | None = None
@@ -87,6 +89,7 @@ class DecisionBody(BaseModel):
class VersionBody(BaseModel): class VersionBody(BaseModel):
title_raw: str title_raw: str
source_photos: str source_photos: str
row_ix: int | None = None
action: str # "pick" | "unknown" action: str # "pick" | "unknown"
version_id: int | None = None version_id: int | None = None
@@ -94,6 +97,7 @@ class VersionBody(BaseModel):
class VetoBody(BaseModel): class VetoBody(BaseModel):
title_raw: str title_raw: str
source_photos: str source_photos: str
row_ix: int | None = None
class DismissBody(BaseModel): class DismissBody(BaseModel):
@@ -112,7 +116,11 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
client=client, client=client,
) )
thumbnails = load_thumbnails(cfg.cache_dir) thumbnails = load_thumbnails(cfg.cache_dir)
dismissed = DismissStore(cfg.data_dir / "unidentified_dismissed.json") dismissed = DismissStore(cfg.dismissed_path)
# FastAPI runs sync endpoints in a threadpool: without this, a polled
# GET /api/state can freshen()-swap session.rows out from under a
# concurrent decision POST, silently dropping the decision.
lock = threading.Lock()
def freshen() -> None: def freshen() -> None:
"""Serve every request from the current file state: an extract or """Serve every request from the current file state: an extract or
@@ -121,8 +129,16 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
if session.reload_if_changed(): if session.reload_if_changed():
thumbnails = load_thumbnails(cfg.cache_dir) thumbnails = load_thumbnails(cfg.cache_dir)
def find_row(title_raw: str, source_photos: str) -> dict: def find_row(
title_raw: str, source_photos: str, row_ix: int | None = None
) -> dict:
freshen() freshen()
# ordinal first: (title_raw, source_photos) is not unique when one
# photo holds two editions of the same game
if row_ix is not None and 0 <= row_ix < len(session.rows):
row = session.rows[row_ix]
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
return row
for row in session.rows: for row in session.rows:
if row["title_raw"] == title_raw and row["source_photos"] == source_photos: if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
return row return row
@@ -134,12 +150,13 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
return {p.name for p in cfg.photos_dir.iterdir() if p.is_file()} return {p.name for p in cfg.photos_dir.iterdir() if p.is_file()}
def row_payload(row: dict) -> dict: def row_payload(row: dict) -> dict:
entry = session.cues_for(row["title_raw"]) entry = session.cues_for(row["title_raw"], row["source_photos"])
available = photo_names() available = photo_names()
candidates = json.loads(row["candidates_json"] or "[]") candidates = json.loads(row["candidates_json"] or "[]")
for c in candidates: for c in candidates:
c["thumbnail"] = thumbnails.get(c.get("bgg_id")) c["thumbnail"] = thumbnails.get(c.get("bgg_id"))
return { return {
"row_ix": session.rows.index(row),
"title_raw": row["title_raw"], "title_raw": row["title_raw"],
"source_photos": row["source_photos"], "source_photos": row["source_photos"],
"match_status": row["match_status"], "match_status": row["match_status"],
@@ -156,6 +173,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
def version_payload(row: dict) -> dict: def version_payload(row: dict) -> dict:
return { return {
"row_ix": session.rows.index(row),
"title_raw": row["title_raw"], "title_raw": row["title_raw"],
"source_photos": row["source_photos"], "source_photos": row["source_photos"],
"bgg_name": row["bgg_name"], "bgg_name": row["bgg_name"],
@@ -164,9 +182,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
def state() -> dict: def state() -> dict:
freshen() freshen()
counts: dict[str, int] = {} counts = Counter(row["match_status"] for row in session.rows)
for row in session.rows:
counts[row["match_status"]] = counts.get(row["match_status"], 0) + 1
resolved_titles = {r["title_raw"] for r in session.rows} resolved_titles = {r["title_raw"] for r in session.rows}
rows_by_title: dict[str, dict] = {} rows_by_title: dict[str, dict] = {}
for r in session.rows: for r in session.rows:
@@ -216,6 +232,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
for r in session.merged_rows() for r in session.merged_rows()
] ]
return { return {
"warnings": session.warnings[-10:],
"pending": [row_payload(r) for r in session.pending_rows()], "pending": [row_payload(r) for r in session.pending_rows()],
"versions": [version_payload(r) for r in session.version_rows()], "versions": [version_payload(r) for r in session.version_rows()],
"merges": merges, "merges": merges,
@@ -240,11 +257,16 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.get("/api/state") @app.get("/api/state")
def api_state() -> dict: def api_state() -> dict:
with lock:
return state() return state()
@app.post("/api/decision") @app.post("/api/decision")
def api_decision(body: DecisionBody) -> dict: def api_decision(body: DecisionBody) -> dict:
row = find_row(body.title_raw, body.source_photos) with lock:
return _decide(body)
def _decide(body: DecisionBody) -> dict:
row = find_row(body.title_raw, body.source_photos, body.row_ix)
if body.action == "pick": if body.action == "pick":
candidates = json.loads(row["candidates_json"] or "[]") candidates = json.loads(row["candidates_json"] or "[]")
chosen = next( chosen = next(
@@ -265,7 +287,11 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.post("/api/version") @app.post("/api/version")
def api_version(body: VersionBody) -> dict: def api_version(body: VersionBody) -> dict:
row = find_row(body.title_raw, body.source_photos) with lock:
return _version(body)
def _version(body: VersionBody) -> dict:
row = find_row(body.title_raw, body.source_photos, body.row_ix)
if body.action == "unknown": if body.action == "unknown":
session.decide_version(row, None) session.decide_version(row, None)
elif body.action == "pick": elif body.action == "pick":
@@ -279,7 +305,8 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.post("/api/veto-merge") @app.post("/api/veto-merge")
def api_veto_merge(body: VetoBody) -> dict: def api_veto_merge(body: VetoBody) -> dict:
row = find_row(body.title_raw, body.source_photos) with lock:
row = find_row(body.title_raw, body.source_photos, body.row_ix)
if row["match_status"] != "merged": if row["match_status"] != "merged":
raise HTTPException(400, "row is not merged") raise HTTPException(400, "row is not merged")
session.veto_merge(row) session.veto_merge(row)
@@ -287,11 +314,14 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
@app.post("/api/dismiss") @app.post("/api/dismiss")
def api_dismiss(body: DismissBody) -> dict: def api_dismiss(body: DismissBody) -> dict:
dismissed.add(_sighting_key(body.photo, body.model_dump(exclude={"photo"}))) with lock:
dismissed.add(
_sighting_key(body.photo, body.model_dump(exclude={"photo"}))
)
return state() return state()
@app.get("/photos/{name}") @app.get("/photos/{name}")
def photo(name: str): def photo(name: str) -> FileResponse:
if name not in photo_names(): # also blocks any path traversal if name not in photo_names(): # also blocks any path traversal
raise HTTPException(404, "no such photo") raise HTTPException(404, "no such photo")
return FileResponse(cfg.photos_dir / name) return FileResponse(cfg.photos_dir / name)
@@ -303,7 +333,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
} }
@app.get("/static/{name}") @app.get("/static/{name}")
def static_asset(name: str): def static_asset(name: str) -> Response:
media_type = _STATIC.get(name) # allowlist: no traversal possible media_type = _STATIC.get(name) # allowlist: no traversal possible
if media_type is None: if media_type is None:
raise HTTPException(404, "no such asset") raise HTTPException(404, "no such asset")
+51
View File
@@ -131,3 +131,54 @@ def test_401_raises_actionable_auth_error(tmp_path, monkeypatch):
client, _, _ = make_client(tmp_path, [(401, "Unauthorized")]) client, _, _ = make_client(tmp_path, [(401, "Unauthorized")])
with pytest.raises(BGGAuthError, match="BGG_API_TOKEN"): with pytest.raises(BGGAuthError, match="BGG_API_TOKEN"):
client.get_xml("search", {"query": "catan"}) client.get_xml("search", {"query": "catan"})
# -- audit-fix regressions ----------------------------------------------
def test_http_200_error_document_raises_and_is_never_cached(tmp_path):
# BGG serves some errors as HTTP 200 <errors> XML; caching one would
# poison every future run for that query
from bggpipe.models import BGGResponseError
errors_xml = "<errors><error><message>Invalid username</message></error></errors>"
client, _, _ = make_client(tmp_path, [(200, errors_xml)])
with pytest.raises(BGGResponseError):
client.get_xml("collection", {"username": "nobody", "own": "1"})
assert list((tmp_path / "cache").glob("*.xml")) == []
def test_collection_full_merges_and_dedupes_by_collid(tmp_path):
base_xml = (
'<items totalitems="2">'
'<item objectid="13" collid="100" subtype="boardgame">'
'<name>Catan</name><status own="1"/></item>'
'<item objectid="177" collid="101" subtype="boardgame">'
'<name>Advanced Civilization</name><status own="1"/></item>'
"</items>"
)
expansion_xml = (
'<items totalitems="1">'
'<item objectid="177" collid="101" subtype="boardgameexpansion">'
'<name>Advanced Civilization</name><status own="1"/></item>'
"</items>"
)
client, _, _ = make_client(tmp_path, [(200, base_xml), (200, expansion_xml)])
items = client.collection_full("eric")
assert len(items) == 2 # collid 101 appears in both responses: one copy
assert {i.coll_id for i in items} == {100, 101}
def test_collection_item_missing_collid_refuses_to_parse(tmp_path):
# a truncated response must fail loudly, not coerce ids to 0 and let
# the dedupe silently drop owned games
from bggpipe.models import BGGResponseError, parse_collection
bad_xml = (
'<items totalitems="1">'
'<item objectid="13" subtype="boardgame">'
'<name>Catan</name><status own="1"/></item>'
"</items>"
)
with pytest.raises(BGGResponseError):
parse_collection(bad_xml)
+15
View File
@@ -1,3 +1,7 @@
"""Config loading: toml knobs, env-only username, unknown-key warning."""
from __future__ import annotations
from pathlib import Path from pathlib import Path
from bggpipe.config import Config, load_config from bggpipe.config import Config, load_config
@@ -29,3 +33,14 @@ def test_username_comes_from_env_only(tmp_path, monkeypatch):
assert load_config(p).bgg_username == "from_env" assert load_config(p).bgg_username == "from_env"
monkeypatch.delenv("BGG_USERNAME") monkeypatch.delenv("BGG_USERNAME")
assert load_config(p).bgg_username == "" assert load_config(p).bgg_username == ""
def test_unknown_toml_keys_warn(tmp_path, monkeypatch):
# a typo'd knob must not silently fall back to defaults
import pytest
monkeypatch.delenv("BGG_USERNAME", raising=False)
p = tmp_path / "config.toml"
p.write_text('photo_dir = "oops"\n')
with pytest.warns(UserWarning, match="photo_dir"):
load_config(p)
+64 -5
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
from pathlib import Path from pathlib import Path
from bggpipe.config import Config
from bggpipe.diff import compute_diff, load_snapshot_collection from bggpipe.diff import compute_diff, load_snapshot_collection
from bggpipe.models import CollectionItem from bggpipe.models import CollectionItem
@@ -114,7 +115,10 @@ def test_owned_with_matching_version_is_just_owned():
assert not result.to_update and not result.to_add assert not result.to_update and not result.to_add
def test_owned_with_different_version_reports_disagreement_untouched(): def test_confident_version_matching_no_copy_is_a_second_copy_to_add():
# Spec: a (bgg_id, version_id) pair is owned only if a collection item
# matches BOTH. All copies carry different versions -> this is an
# additional physical copy; existing entries are never edited.
result = compute_diff( result = compute_diff(
[ [
_match( _match(
@@ -127,9 +131,10 @@ def test_owned_with_different_version_reports_disagreement_untouched():
], ],
[_item(266192, 5, version_id=465063)], [_item(266192, 5, version_id=465063)],
) )
assert result.already_owned == ["Wingspan"]
assert not result.to_update # additive only: never edit a set version assert not result.to_update # additive only: never edit a set version
assert "fourth printing" in result.disagreements[0] assert [r["version_id"] for r in result.to_add] == ["521212"]
assert "fourth printing" in result.second_copies[0]
assert result.already_owned == []
def test_version_unknown_owned_by_bare_id(): def test_version_unknown_owned_by_bare_id():
@@ -169,8 +174,11 @@ def test_pending_rejected_and_unseen_are_reported():
def test_merged_rows_are_skipped_but_photos_carry_to_survivor(): def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
matches = [ matches = [
_match("Joking Hazard", "193621"), _match("Joking Hazard", "193621"),
{**_match("Jokin Ha...", "193621", status="merged"), {
"merged_into": "Joking Hazard", "source_photos": "other.jpg"}, **_match("Jokin Ha...", "193621", status="merged"),
"merged_into": "Joking Hazard",
"source_photos": "other.jpg",
},
] ]
result = compute_diff(matches, []) # empty collection -> to_add result = compute_diff(matches, []) # empty collection -> to_add
assert result.merged == 1 assert result.merged == 1
@@ -178,3 +186,54 @@ def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
(row,) = result.to_add (row,) = result.to_add
assert row["title_raw"] == "Joking Hazard" assert row["title_raw"] == "Joking Hazard"
assert row["source_photos"] == "other.jpg;x.jpg" # combined assert row["source_photos"] == "other.jpg;x.jpg" # combined
def test_versionless_copies_exhaust_then_second_copy_becomes_add():
# two confident-version matches, ONE versionless copy: the first consumes
# it (to_update), the second is an additional physical copy (to_add)
result = compute_diff(
[
_match("Sorcerer", "39", vstatus="version_auto", vid="111", vname="1st"),
_match("Sorcerer", "39", vstatus="version_auto", vid="222", vname="2nd"),
],
[_item(39, 701)],
)
assert [u["version_id"] for u in result.to_update] == ["111"]
assert [a["version_id"] for a in result.to_add] == ["222"]
assert len(result.second_copies) == 1
def test_run_diff_outputs_feed_upload_unchanged(tmp_path, monkeypatch):
# the cross-stage contract: whatever run_diff writes, run_upload must
# read — a column rename on either side has to fail HERE
import shutil
from bggpipe.diff import run_diff
from bggpipe.resolve import write_matches
from bggpipe.upload import run_upload
monkeypatch.delenv("BGG_API_TOKEN", raising=False)
cfg = Config(data_dir=tmp_path)
fixtures = Path(__file__).parent / "fixtures"
for name in (
"collection_snapshot_base.xml",
"collection_snapshot_expansions.xml",
):
shutil.copy(fixtures / name, tmp_path / name)
write_matches(
cfg.matches_path,
[
_match("Wingspan", "266192", status="auto"), # not in the snapshots
_match("5 MINUTE DUNGEON", "207830", status="auto"), # owned
],
)
result = run_diff(cfg)
assert [r["bgg_id"] for r in result.to_add] == ["266192"]
from test_upload import FakeUploader
fake = FakeUploader()
run_upload(cfg, uploader=fake, sleep=lambda s: None, now=lambda: "t")
assert [(j.action, j.bgg_id, j.name) for j in fake.calls] == [
("add", "266192", "Wingspan")
]
+5 -5
View File
@@ -5,6 +5,7 @@ from __future__ import annotations
import base64 import base64
import io import io
import json
import pytest import pytest
import typer import typer
@@ -83,21 +84,21 @@ def test_parse_object_with_titles_and_unidentified():
"unidentified": [{"location": "top shelf, left of Catan", "unidentified": [{"location": "top shelf, left of Catan",
"partial_text": "WAR", "art_notes": "red spine"}]} "partial_text": "WAR", "art_notes": "red spine"}]}
```""" ```"""
titles, unidentified = parse_vision_response(text) titles, unidentified, _ = parse_vision_response(text)
assert titles[0]["title_raw"] == "Catan" assert titles[0]["title_raw"] == "Catan"
assert unidentified[0]["location"] == "top shelf, left of Catan" assert unidentified[0]["location"] == "top shelf, left of Catan"
def test_parse_legacy_bare_array_still_works(): def test_parse_legacy_bare_array_still_works():
text = '```json\n[{"title_raw": "Catan", "confidence": "high"}]\n```' text = '```json\n[{"title_raw": "Catan", "confidence": "high"}]\n```'
titles, unidentified = parse_vision_response(text) titles, unidentified, _ = parse_vision_response(text)
assert titles[0]["title_raw"] == "Catan" assert titles[0]["title_raw"] == "Catan"
assert unidentified == [] assert unidentified == []
def test_parse_tolerates_prose_around_json(): def test_parse_tolerates_prose_around_json():
text = 'Here are the games:\n[{"title_raw": "Wingspan"}]\nLet me know!' text = 'Here are the games:\n[{"title_raw": "Wingspan"}]\nLet me know!'
titles, _ = parse_vision_response(text) titles, _, _ = parse_vision_response(text)
assert titles[0]["title_raw"] == "Wingspan" assert titles[0]["title_raw"] == "Wingspan"
@@ -106,7 +107,7 @@ def test_parse_drops_malformed_entries():
'{"titles": [{"title_raw": "Catan"}, {"no_title": true}, "just a string"],' '{"titles": [{"title_raw": "Catan"}, {"no_title": true}, "just a string"],'
' "unidentified": [{}, "not a dict", {"location": "somewhere"}]}' ' "unidentified": [{}, "not a dict", {"location": "somewhere"}]}'
) )
titles, unidentified = parse_vision_response(text) titles, unidentified, _ = parse_vision_response(text)
assert len(titles) == 1 assert len(titles) == 1
assert len(unidentified) == 1 # empty {} and the bare string are dropped assert len(unidentified) == 1 # empty {} and the bare string are dropped
@@ -227,7 +228,6 @@ def test_run_extract_empty_photos_dir_exits(tmp_path):
# -- unidentified sightings --------------------------------------------- # -- unidentified sightings ---------------------------------------------
import json # noqa: E402
OBJECT_PAYLOAD = json.dumps( OBJECT_PAYLOAD = json.dumps(
{ {
+4
View File
@@ -1,3 +1,7 @@
"""XML parser tests against hand-built API2 response shapes."""
from __future__ import annotations
import pytest import pytest
from bggpipe.models import ( from bggpipe.models import (
+4
View File
@@ -1,3 +1,7 @@
"""Title normalization must be symmetric and aggressive (spec)."""
from __future__ import annotations
from bggpipe.normalize import normalize_title from bggpipe.normalize import normalize_title
+120 -12
View File
@@ -14,17 +14,23 @@ from pathlib import Path
import httpx import httpx
import pytest import pytest
from bggpipe.bgg_client import BGGClient from bggpipe.bgg_client import BGGClient, cache_key
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.models import GameVersion from bggpipe.models import GameVersion
from bggpipe.normalize import normalize_title
from bggpipe.resolve import ( from bggpipe.resolve import (
Candidate, Candidate,
MatchRow,
TitleEntry, TitleEntry,
_dominant, _dominant,
_score_version, _score_version,
_truncation_heads,
dedupe_matches,
load_titles, load_titles,
read_matches,
resolve_entry, resolve_entry,
run_resolve, run_resolve,
write_matches,
) )
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache" FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
@@ -191,9 +197,6 @@ def test_score_version_no_overlap():
# -- progressive title truncation (long transcribed box titles) --------- # -- progressive title truncation (long transcribed box titles) ---------
from bggpipe.normalize import normalize_title # noqa: E402
from bggpipe.resolve import _truncation_heads # noqa: E402
CIV_TITLE = ( CIV_TITLE = (
"CIVILIZATION Game of the Heroic Age - The Dawn of History 8000 BC to 250 BC" "CIVILIZATION Game of the Heroic Age - The Dawn of History 8000 BC to 250 BC"
) )
@@ -347,13 +350,15 @@ def test_run_resolve_saves_progress_when_token_missing(tmp_path):
# -- post-resolve dedupe ------------------------------------------------ # -- post-resolve dedupe ------------------------------------------------
from bggpipe.bgg_client import cache_key as _cache_key # noqa: E402
from bggpipe.resolve import dedupe_matches # noqa: E402
def _mrow( def _mrow(
title, bgg_id, photos, name="Joking Hazard", title,
vstatus="version_unknown", vid="", status="auto", bgg_id,
photos,
name="Joking Hazard",
vstatus="version_unknown",
vid="",
status="auto",
): ):
return { return {
"title_raw": title, "title_raw": title,
@@ -446,8 +451,6 @@ def test_dedupe_is_idempotent_and_skips_merged():
def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path): def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
from bggpipe.resolve import write_matches as _wm # noqa: F401
cache = tmp_path / "cache" cache = tmp_path / "cache"
cache.mkdir() cache.mkdir()
wingspan_xml = ( wingspan_xml = (
@@ -456,7 +459,7 @@ def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
"</item></items>" "</item></items>"
) )
for query in ("Wingspan", "WINGSPAN!"): for query in ("Wingspan", "WINGSPAN!"):
key = _cache_key( key = cache_key(
"search", {"query": query, "type": "boardgame,boardgameexpansion"} "search", {"query": query, "type": "boardgame,boardgameexpansion"}
) )
(cache / key).write_text(wingspan_xml) (cache / key).write_text(wingspan_xml)
@@ -482,3 +485,108 @@ def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
assert saved["Wingspan"]["match_status"] == "auto" assert saved["Wingspan"]["match_status"] == "auto"
assert saved["WINGSPAN!"]["match_status"] == "merged" assert saved["WINGSPAN!"]["match_status"] == "merged"
assert saved["WINGSPAN!"]["merged_into"] == "Wingspan" assert saved["WINGSPAN!"]["merged_into"] == "Wingspan"
# -- audit-fix regressions ----------------------------------------------
def test_run_resolve_force_rebuilds_from_scratch(client, tmp_path):
data_dir = tmp_path / "data"
data_dir.mkdir()
shutil.copy(TITLES_JSON, data_dir / "titles.json")
cfg = Config(data_dir=data_dir)
run_resolve(cfg, client=client)
# poison one row: force must throw it away and re-resolve everything
rows = read_matches(cfg.matches_path)
rows[0]["match_status"] = "rejected"
write_matches(cfg.matches_path, rows)
forced = run_resolve(cfg, force=True, client=client)
assert len(forced) == 7 # every title re-resolved, none skipped
fresh = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
assert fresh["Catan"]["match_status"] == "auto"
def test_new_photo_of_resolved_game_updates_row_instead_of_duplicating(
client, tmp_path
):
data_dir = tmp_path / "data"
data_dir.mkdir()
shutil.copy(TITLES_JSON, data_dir / "titles.json")
cfg = Config(data_dir=data_dir)
run_resolve(cfg, client=client)
n_rows = len(read_matches(cfg.matches_path))
# extract sees Catan again on a reshoot photo: the entry's photo set
# grows, its (title, photos) key changes
titles = json.loads((data_dir / "titles.json").read_text())
for entry in titles:
if entry["title_raw"] == "Catan":
entry["source_photos"] = sorted([*entry["source_photos"], "reshoot.jpg"])
(data_dir / "titles.json").write_text(json.dumps(titles))
assert run_resolve(cfg, client=client) == [] # nothing re-resolved
rows = read_matches(cfg.matches_path)
assert len(rows) == n_rows # and no duplicate row appended
catan = next(r for r in rows if r["title_raw"] == "Catan")
assert "reshoot.jpg" in catan["source_photos"] # provenance followed
def test_dedupe_never_overturns_a_human_veto():
a = _mrow("CATAN", "13", "p1.jpg")
b = _mrow("Catan", "13", "p2.jpg", status="approved")
b["dedupe_veto"] = "1" # review said: genuinely two copies
events = dedupe_matches([a, b], [])
assert events == []
assert b["match_status"] == "approved"
def test_publisher_pick_refuses_multiple_same_publisher_candidates():
from bggpipe.resolve import _publisher_pick
entry = TitleEntry(
title_raw="Sorcerer", title_normalized="sorcerer", publisher_hint="SPI"
)
cands = [
_cand(1, exact=True),
_cand(2, exact=True),
]
for c in cands:
c.publishers = ["Simulations Publications, Inc. (SPI)"]
assert _publisher_pick(entry, cands) is None
def test_publisher_pick_refuses_mixed_base_and_expansion():
from bggpipe.resolve import _publisher_pick
entry = TitleEntry(
title_raw="Wingspan", title_normalized="wingspan", publisher_hint="Stonemaier"
)
base = _cand(1, exact=True, type_="boardgame")
expansion = _cand(2, exact=True, type_="boardgameexpansion")
for c in (base, expansion):
c.publishers = ["Stonemaier Games"]
assert _publisher_pick(entry, [base, expansion]) is None
def test_resolve_version_handles_unknown_id():
from bggpipe.resolve import resolve_version
class EmptyThings:
def things(self, ids, **kwargs):
return []
entry = TitleEntry(title_raw="X", title_normalized="x", publisher_hint="Someone")
row = MatchRow(title_raw="X", bgg_id=999999)
resolve_version(EmptyThings(), entry, row) # must not raise
assert row.version_status == "version_unknown"
def test_empty_normalized_title_never_matches(client):
# 风声 normalizes to "" — empty-vs-empty must not count as exact
from bggpipe.resolve import _plausible_candidates
entry = TitleEntry(title_raw="风声", title_normalized="")
# any cached query works; candidates must be rejected regardless of name
assert _plausible_candidates(client, entry, "Catan") == []
+95 -1
View File
@@ -9,12 +9,13 @@ import json
from pathlib import Path from pathlib import Path
import httpx import httpx
import pytest
from rich.console import Console from rich.console import Console
from bggpipe.bgg_client import BGGClient from bggpipe.bgg_client import BGGClient
from bggpipe.config import Config from bggpipe.config import Config
from bggpipe.resolve import read_matches, write_matches from bggpipe.resolve import read_matches, write_matches
from bggpipe.review import run_review from bggpipe.review import ReviewSession, run_review
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache" FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
@@ -288,3 +289,96 @@ def test_version_pass_is_skippable(tmp_path):
) )
(row,) = read_matches(cfg.matches_path) (row,) = read_matches(cfg.matches_path)
assert row["version_status"] == "version_ambiguous" # untouched, review later assert row["version_status"] == "version_ambiguous" # untouched, review later
# -- audit-fix regressions ----------------------------------------------
def test_veto_merge_persists_against_future_dedupe(tmp_path):
from bggpipe.resolve import read_matches
cfg = _setup(
tmp_path,
[
_row(title_raw="CATAN", match_status="merged", merged_into="Catan"),
_row(title_raw="Catan", match_status="auto", bgg_id="13"),
],
)
session = ReviewSession(
cfg,
console=quiet_console(),
input_fn=scripted(),
client=unauthorized_client(tmp_path),
)
merged = next(r for r in session.rows if r["match_status"] == "merged")
session.veto_merge(merged)
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
assert saved["CATAN"]["match_status"] == "approved"
assert saved["CATAN"]["dedupe_veto"] == "1" # survives resolve re-runs
def test_failed_save_never_leaves_memory_ahead_of_disk(tmp_path, monkeypatch):
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
session = ReviewSession(
cfg,
console=quiet_console(),
input_fn=scripted(),
client=unauthorized_client(tmp_path),
)
row = session.rows[0]
import bggpipe.review as review_mod
def exploding_write(path, rows):
raise OSError("disk full")
monkeypatch.setattr(review_mod, "write_matches", exploding_write)
with pytest.raises(OSError):
session.decide_reject(row)
# memory was rolled back to what disk actually holds
assert session.rows[0]["match_status"] == "unmatched"
assert session.decisions == 0
def test_save_merges_own_decision_over_concurrent_external_rewrite(tmp_path):
from bggpipe.resolve import read_matches, write_matches
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
session = ReviewSession(
cfg,
console=quiet_console(),
input_fn=scripted(),
client=unauthorized_client(tmp_path),
)
row = session.rows[0]
# resolve appends a new row in another terminal AFTER our session loaded
external = read_matches(cfg.matches_path)
external.append(_row(title_raw="Newcomer", match_status="auto", bgg_id="7"))
write_matches(cfg.matches_path, external)
session.decide_reject(row)
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
assert saved["Mystery"]["match_status"] == "rejected" # our decision
assert "Newcomer" in saved # their row survived too
def test_manual_id_unknown_to_bgg_warns_instead_of_crashing(tmp_path):
# an empty /thing result (mistyped id) used to crash the whole session
import httpx as _httpx
empty_things = BGGClient(
cache_dir=tmp_path / "cache",
transport=_httpx.MockTransport(
lambda req: _httpx.Response(200, text='<items total="0"></items>')
),
sleep=lambda s: None,
)
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
session = ReviewSession(
cfg, console=quiet_console(), input_fn=scripted(), client=empty_things
)
session.decide_manual(session.rows[0], 999999)
assert session.rows[0]["match_status"] == "approved"
assert session.rows[0]["bgg_id"] == "999999"
assert any("no game with id 999999" in w for w in session.warnings)
+88 -5
View File
@@ -111,7 +111,7 @@ class FakeUploader:
def test_build_queue_skips_logged_successes(): def test_build_queue_skips_logged_successes():
jobs, done, failed = build_queue( jobs, done, failed, _ = build_queue(
[_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")], [_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")],
[_update_row(collid="9")], [_update_row(collid="9")],
[ [
@@ -127,7 +127,7 @@ def test_build_queue_skips_logged_successes():
def test_build_queue_second_copy_is_a_distinct_job(): def test_build_queue_second_copy_is_a_distinct_job():
# Same game, different version: a separate physical copy, so a # Same game, different version: a separate physical copy, so a
# logged add of one version must not swallow the other. # logged add of one version must not swallow the other.
jobs, done, _ = build_queue( jobs, done, _, _ = build_queue(
[ [
_add_row(bgg_id="1", version_id="10", version_name="First ed."), _add_row(bgg_id="1", version_id="10", version_name="First ed."),
_add_row(bgg_id="1", version_id="11", version_name="Second ed."), _add_row(bgg_id="1", version_id="11", version_name="Second ed."),
@@ -141,9 +141,11 @@ def test_build_queue_second_copy_is_a_distinct_job():
def test_build_queue_failures_need_retry_flag(): def test_build_queue_failures_need_retry_flag():
log = [_log_row(action="add", bgg_id="1", status="failed")] log = [_log_row(action="add", bgg_id="1", status="failed")]
jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log) jobs, _, skipped, _ = build_queue([_add_row(bgg_id="1")], [], log)
assert jobs == [] and skipped == 1 assert jobs == [] and skipped == 1
jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True) jobs, _, skipped, _ = build_queue(
[_add_row(bgg_id="1")], [], log, retry_failed=True
)
assert len(jobs) == 1 and skipped == 0 assert len(jobs) == 1 and skipped == 0
@@ -153,7 +155,7 @@ def test_build_queue_latest_log_entry_wins():
_log_row(action="add", bgg_id="1", status="failed"), _log_row(action="add", bgg_id="1", status="failed"),
_log_row(action="add", bgg_id="1", status="added"), _log_row(action="add", bgg_id="1", status="added"),
] ]
jobs, done, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True) jobs, done, _, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
assert jobs == [] and done == 1 assert jobs == [] and done == 1
@@ -300,3 +302,84 @@ def test_fresh_clone_marker_blocks_upload_without_cache_dir(tmp_path):
_seed_data(tmp_path, to_add=[_add_row()]) _seed_data(tmp_path, to_add=[_add_row()])
with pytest.raises(typer.Exit): with pytest.raises(typer.Exit):
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW) run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
# -- audit-fix regressions ----------------------------------------------
def test_login_error_aborts_without_poisoning_the_log(tmp_path):
from bggpipe.upload import LoginError
class BrokenLogin(FakeUploader):
def add_game(self, job):
raise LoginError("Cloudflare is challenging this browser")
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 4)])
results = run_upload(cfg, uploader=BrokenLogin(), sleep=lambda s: None, now=NOW)
assert results == [] # nothing logged: next run retries everything
assert not (tmp_path / "upload_log.csv").exists()
def test_three_identical_failures_abort_as_systemic(tmp_path):
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 6)])
fake = FakeUploader(failures={"Wingspan"}) # every job shares the name
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
assert len(results) == 3 # aborted after the third identical failure
logged = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
assert len(logged) == 3 # jobs 4-5 left unlogged and retryable
def test_added_no_version_is_done_and_verify_tolerates_it(tmp_path):
class NoVersionPicker(FakeUploader):
def add_game(self, job):
self.calls.append(job)
return "added_no_version", "version not in picker; added without version"
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row(version_id="99", version_name="4th ed.")])
run_upload(cfg, uploader=NoVersionPicker(), sleep=lambda s: None, now=NOW)
# done: re-running must NOT re-add (a duplicate collection entry)
again = FakeUploader()
assert run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW) == []
assert again.calls == []
# verify: game present without the version is the EXPECTED outcome
log = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
assert verify_uploads(log, [_item(1, 10)]) == []
def test_missing_to_add_csv_is_a_loud_precondition_failure(tmp_path):
cfg = _cfg(tmp_path) # no diff outputs seeded at all
with pytest.raises(typer.Exit):
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
def test_second_update_for_same_game_is_deferred(tmp_path):
# the row-edit flow can't target a collid, so only one update per game
# per run is safe
cfg = _cfg(tmp_path)
_seed_data(
tmp_path,
to_update=[
_update_row(collid="9", bgg_id="2"),
_update_row(collid="10", bgg_id="2", vid="26", vname="2nd ed."),
],
)
fake = FakeUploader()
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
assert [r["collid"] for r in results] == ["9"]
# after the first lands, the next run picks up the deferred one
again = FakeUploader()
results = run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW)
assert [j.collid for j in again.calls] == ["10"]
def test_real_run_without_credentials_exits_before_any_browser(tmp_path, monkeypatch):
monkeypatch.delenv("BGG_USERNAME", raising=False)
monkeypatch.delenv("BGG_PASSWORD", raising=False)
cfg = _cfg(tmp_path)
_seed_data(tmp_path, to_add=[_add_row()])
with pytest.raises(typer.Exit):
run_upload(cfg, sleep=lambda s: None, now=NOW) # uploader=None: real path
assert not (tmp_path / "upload_log.csv").exists()
+20
View File
@@ -383,3 +383,23 @@ def test_own_saves_do_not_count_as_external_changes(tmp_path):
write_matches(cfg.matches_path, session.rows + [_row(title_raw="X")]) write_matches(cfg.matches_path, session.rows + [_row(title_raw="X")])
assert session.reload_if_changed() is True # someone else's assert session.reload_if_changed() is True # someone else's
assert any(r["title_raw"] == "X" for r in session.rows) assert any(r["title_raw"] == "X" for r in session.rows)
def test_session_warnings_surface_in_state(tmp_path):
web, cfg = make_client(tmp_path)
# a manual id triggers a lookup against the 401-ing client: the session
# degrades and the warning must reach the payload (the session console
# is a StringIO here — this is the only way the user ever sees it)
res = web.post(
"/api/decision",
json={
"title_raw": "Mystery",
"source_photos": "shelf.jpg",
"action": "manual",
"bgg_id": 42,
},
)
assert res.status_code == 200
warnings = res.json()["warnings"]
assert any("couldn't look up id 42" in w for w in warnings)
assert warnings == web.get("/api/state").json()["warnings"]