Audit: 5-reviewer sweep — 19 fixes across every stage, +24 tests
Correctness: review vetoes persist via a dedupe_veto column (resolve re-runs no longer overturn humans); diff emits second copies whose confident version matches no owned copy (spec: pairs own only on both ids) and fetches the live collection with refresh; resolve pairs titles.json entries to rows by title so a reshoot photo updates provenance instead of duplicating rows; version lookups survive empty /thing results; publisher tie-break now honors the mixed base/expansion veto and refuses multi-candidate picks; empty-normalized (non-Latin) titles never count as exact. Upload: LoginError aborts a run instead of logging N bogus failures (and 3 identical consecutive failures abort as systemic); Cloudflare interstitials are detected; added-without-version gets its own logged status that verify understands; same-game updates run one per pass so the name-targeted row edit can't overwrite a fresh version; absent diff outputs fail loudly; pagination clicks are paced. Web review: a lock serializes freshen/decide (threadpool race dropped decisions); failed saves roll memory back and always alert the browser (non-JSON 500s included); session warnings reach the page instead of a StringIO; state-load failures and dead servers show banners instead of a blank page; duplicate (title, photos) rows are addressable by ordinal. Consistency: shared CONFIDENT_VERSION_STATUSES, client_for(), Config paths for every artifact, one review-port constant, named matching thresholds, strict collection-id parsing, error-doc responses never cached, unknown config keys warn, extract reports dropped vision entries, fixture generators share escaping + marker text. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,7 +21,7 @@ Full design lives in `bgg-shelf-pipeline-spec.md` (read it before changing pipel
|
|||||||
|
|
||||||
- `uv sync` — install deps (Python 3.12+, managed by **uv**; use `uv add`, never pip). Playwright needs a one-time `uv run playwright install chromium`.
|
- `uv sync` — install deps (Python 3.12+, managed by **uv**; use `uv add`, never pip). Playwright needs a one-time `uv run playwright install chromium`.
|
||||||
- `uv run bggpipe <stage>` — run a pipeline stage. Non-secret settings come from `config.toml` (username, dirs, vision model, rate limit); `--config` overrides the path.
|
- `uv run bggpipe <stage>` — run a pipeline stage. Non-secret settings come from `config.toml` (username, dirs, vision model, rate limit); `--config` overrides the path.
|
||||||
- `uv run pytest` — 105 tests, all offline against fixtures. Tests marked `live` hit the real BGG API (read-only) and are skipped unless you pass `--run-live`.
|
- `uv run pytest` — the suite runs fully offline against fixtures. Tests marked `live` hit the real BGG API (read-only) and are skipped unless you pass `--run-live`.
|
||||||
- `uv run ruff check` / `uv run ruff format` — lint (rules E, F, I, UP, B, SIM) and format.
|
- `uv run ruff check` / `uv run ruff format` — lint (rules E, F, I, UP, B, SIM) and format.
|
||||||
|
|
||||||
## Layout
|
## Layout
|
||||||
|
|||||||
@@ -0,0 +1,36 @@
|
|||||||
|
"""Shared plumbing for the two stub-fixture generators.
|
||||||
|
|
||||||
|
Both write into the same cache dirs, so they must agree on XML escaping
|
||||||
|
(a title containing & or " must not produce malformed XML) and on the
|
||||||
|
provenance-marker text the upload guard depends on.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
CACHE_MARKER_TEXT = (
|
||||||
|
"This cache contains hand-written stub XML, not real BGG "
|
||||||
|
"responses. Data resolved from it must not be uploaded.\n"
|
||||||
|
)
|
||||||
|
DATA_MARKER_TEXT = (
|
||||||
|
"The CSVs in this directory were resolved from hand-written stub "
|
||||||
|
"fixtures, not real BGG data — version_ids are SYNTHETIC. The "
|
||||||
|
"upload stage refuses to run while this file exists. Delete it "
|
||||||
|
"only after re-resolving against real recorded fixtures "
|
||||||
|
"(BGG_API_TOKEN + scripts/record_fixtures.py + resolve --force).\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def esc(text: str) -> str:
|
||||||
|
"""Minimal XML attribute/text escaping for hand-built fixture strings."""
|
||||||
|
return str(text).replace("&", "&").replace("<", "<").replace('"', """)
|
||||||
|
|
||||||
|
|
||||||
|
def write_cache_marker(target: Path) -> None:
|
||||||
|
(target / "STUB_FIXTURES.marker").write_text(CACHE_MARKER_TEXT)
|
||||||
|
|
||||||
|
|
||||||
|
def write_data_marker(data_dir: Path = Path("data")) -> None:
|
||||||
|
data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(data_dir / "STUB_DATA.marker").write_text(DATA_MARKER_TEXT)
|
||||||
@@ -17,6 +17,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from fixture_common import esc, write_cache_marker, write_data_marker
|
||||||
|
|
||||||
from bggpipe.bgg_client import cache_key
|
from bggpipe.bgg_client import cache_key
|
||||||
|
|
||||||
TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
|
TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
|
||||||
@@ -295,7 +297,7 @@ VERSIONS: dict[int, list[tuple]] = {
|
|||||||
def search_xml(results: list[tuple]) -> str:
|
def search_xml(results: list[tuple]) -> str:
|
||||||
items = "".join(
|
items = "".join(
|
||||||
f'<item type="{type_}" id="{bgg_id}">'
|
f'<item type="{type_}" id="{bgg_id}">'
|
||||||
f'<name type="{name_type}" value="{_esc(name)}"/>'
|
f'<name type="{name_type}" value="{esc(name)}"/>'
|
||||||
f'<yearpublished value="{year}"/></item>'
|
f'<yearpublished value="{year}"/></item>'
|
||||||
for bgg_id, name, year, type_, name_type in results
|
for bgg_id, name, year, type_, name_type in results
|
||||||
)
|
)
|
||||||
@@ -306,12 +308,12 @@ def stats_xml(things: list[tuple]) -> str:
|
|||||||
items = ""
|
items = ""
|
||||||
for bgg_id, name, year, type_, owned, rank, publishers in things:
|
for bgg_id, name, year, type_, owned, rank, publishers in things:
|
||||||
links = "".join(
|
links = "".join(
|
||||||
f'<link type="boardgamepublisher" id="1" value="{_esc(p)}"/>'
|
f'<link type="boardgamepublisher" id="1" value="{esc(p)}"/>'
|
||||||
for p in publishers
|
for p in publishers
|
||||||
)
|
)
|
||||||
items += (
|
items += (
|
||||||
f'<item type="{type_}" id="{bgg_id}">'
|
f'<item type="{type_}" id="{bgg_id}">'
|
||||||
f'<name type="primary" value="{_esc(name)}"/>'
|
f'<name type="primary" value="{esc(name)}"/>'
|
||||||
f'<yearpublished value="{year}"/>{links}'
|
f'<yearpublished value="{year}"/>{links}'
|
||||||
f"<statistics><ratings>"
|
f"<statistics><ratings>"
|
||||||
f'<owned value="{owned}"/>'
|
f'<owned value="{owned}"/>'
|
||||||
@@ -326,14 +328,14 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
|
|||||||
version_items = ""
|
version_items = ""
|
||||||
for vid, name, year, publishers, languages in versions:
|
for vid, name, year, publishers, languages in versions:
|
||||||
links = "".join(
|
links = "".join(
|
||||||
f'<link type="boardgamepublisher" id="1" value="{_esc(p)}"/>'
|
f'<link type="boardgamepublisher" id="1" value="{esc(p)}"/>'
|
||||||
for p in publishers
|
for p in publishers
|
||||||
) + "".join(
|
) + "".join(
|
||||||
f'<link type="language" id="1" value="{_esc(lang)}"/>' for lang in languages
|
f'<link type="language" id="1" value="{esc(lang)}"/>' for lang in languages
|
||||||
)
|
)
|
||||||
version_items += (
|
version_items += (
|
||||||
f'<item type="boardgameversion" id="{vid}">'
|
f'<item type="boardgameversion" id="{vid}">'
|
||||||
f'<name type="primary" value="{_esc(name)}"/>'
|
f'<name type="primary" value="{esc(name)}"/>'
|
||||||
f'<yearpublished value="{year}"/>{links}</item>'
|
f'<yearpublished value="{year}"/>{links}</item>'
|
||||||
)
|
)
|
||||||
return (
|
return (
|
||||||
@@ -343,9 +345,6 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _esc(text: str) -> str:
|
|
||||||
return text.replace("&", "&").replace("<", "<").replace('"', """)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
files: dict[str, str] = {}
|
files: dict[str, str] = {}
|
||||||
@@ -364,18 +363,8 @@ def main() -> None:
|
|||||||
(target / name).write_text(xml)
|
(target / name).write_text(xml)
|
||||||
# provenance marker: anything resolved from this cache is stub-derived
|
# provenance marker: anything resolved from this cache is stub-derived
|
||||||
# and NOT upload-ready; re-recording real fixtures removes the marker
|
# and NOT upload-ready; re-recording real fixtures removes the marker
|
||||||
(target / "STUB_FIXTURES.marker").write_text(
|
write_cache_marker(target)
|
||||||
"This cache contains hand-written stub XML, not real BGG "
|
write_data_marker(DATA_MARKER.parent)
|
||||||
"responses. Data resolved from it must not be uploaded.\n"
|
|
||||||
)
|
|
||||||
DATA_MARKER.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
DATA_MARKER.write_text(
|
|
||||||
"The CSVs in this directory were resolved from hand-written stub "
|
|
||||||
"fixtures, not real BGG data — version_ids are SYNTHETIC. The "
|
|
||||||
"upload stage refuses to run while this file exists. Delete it "
|
|
||||||
"only after re-resolving against real recorded fixtures "
|
|
||||||
"(BGG_API_TOKEN + scripts/record_fixtures.py + resolve --force).\n"
|
|
||||||
)
|
|
||||||
print(f"Wrote {len(files)} fixture file(s) to {' and '.join(map(str, TARGETS))}")
|
print(f"Wrote {len(files)} fixture file(s) to {' and '.join(map(str, TARGETS))}")
|
||||||
print(f"Wrote {DATA_MARKER} (committed; upload refuses while it exists)")
|
print(f"Wrote {DATA_MARKER} (committed; upload refuses while it exists)")
|
||||||
|
|
||||||
|
|||||||
@@ -14,6 +14,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from fixture_common import esc, write_cache_marker
|
||||||
|
|
||||||
from bggpipe.bgg_client import cache_key
|
from bggpipe.bgg_client import cache_key
|
||||||
|
|
||||||
FIXTURE_CACHE = Path("tests/fixtures/bgg_cache")
|
FIXTURE_CACHE = Path("tests/fixtures/bgg_cache")
|
||||||
@@ -24,7 +26,7 @@ def search_item(bgg_id: int, name: str, year: int | None, type_: str) -> str:
|
|||||||
year_xml = f'<yearpublished value="{year}"/>' if year else ""
|
year_xml = f'<yearpublished value="{year}"/>' if year else ""
|
||||||
return (
|
return (
|
||||||
f'<item type="{type_}" id="{bgg_id}">'
|
f'<item type="{type_}" id="{bgg_id}">'
|
||||||
f'<name type="primary" value="{name}"/>{year_xml}</item>'
|
f'<name type="primary" value="{esc(name)}"/>{year_xml}</item>'
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -104,10 +106,7 @@ THINGS = {
|
|||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
FIXTURE_CACHE.mkdir(parents=True, exist_ok=True)
|
FIXTURE_CACHE.mkdir(parents=True, exist_ok=True)
|
||||||
(FIXTURE_CACHE / "STUB_FIXTURES.marker").write_text(
|
write_cache_marker(FIXTURE_CACHE)
|
||||||
"This cache contains hand-written stub XML, not real BGG "
|
|
||||||
"responses. Data resolved from it must not be uploaded.\n"
|
|
||||||
)
|
|
||||||
for query, items in SEARCHES.items():
|
for query, items in SEARCHES.items():
|
||||||
key = cache_key("search", {"query": query, "type": SEARCH_TYPES})
|
key = cache_key("search", {"query": query, "type": SEARCH_TYPES})
|
||||||
total = items.count("<item ")
|
total = items.count("<item ")
|
||||||
|
|||||||
@@ -18,7 +18,10 @@ from urllib.parse import urlencode
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
|
from bggpipe import __version__
|
||||||
|
from bggpipe.config import Config
|
||||||
from bggpipe.models import (
|
from bggpipe.models import (
|
||||||
|
BGGResponseError,
|
||||||
CollectionItem,
|
CollectionItem,
|
||||||
SearchResult,
|
SearchResult,
|
||||||
ThingDetails,
|
ThingDetails,
|
||||||
@@ -65,7 +68,7 @@ class BGGClient:
|
|||||||
self._monotonic = monotonic
|
self._monotonic = monotonic
|
||||||
self._rng = rng or random.Random()
|
self._rng = rng or random.Random()
|
||||||
self._last_request: float | None = None
|
self._last_request: float | None = None
|
||||||
headers = {"User-Agent": "bggpipe/0.1 (shelf-collection pipeline)"}
|
headers = {"User-Agent": f"bggpipe/{__version__} (shelf-collection pipeline)"}
|
||||||
# BGG requires registered applications since 2025: the token from
|
# BGG requires registered applications since 2025: the token from
|
||||||
# https://boardgamegeek.com/applications must accompany every request.
|
# https://boardgamegeek.com/applications must accompany every request.
|
||||||
# Env var only — never config, disk, or logs.
|
# Env var only — never config, disk, or logs.
|
||||||
@@ -117,6 +120,12 @@ class BGGClient:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
|
if "<errors" in response.text[:120]:
|
||||||
|
# BGG serves some errors as HTTP 200 <errors> XML (bad
|
||||||
|
# username etc.) — caching one would poison every re-run
|
||||||
|
raise BGGResponseError(
|
||||||
|
f"BGG error document for /{endpoint}: {response.text[:200]}"
|
||||||
|
)
|
||||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
||||||
cache_path.write_text(response.text)
|
cache_path.write_text(response.text)
|
||||||
return response.text
|
return response.text
|
||||||
@@ -137,6 +146,7 @@ class BGGClient:
|
|||||||
def things(
|
def things(
|
||||||
self,
|
self,
|
||||||
ids: Iterable[int],
|
ids: Iterable[int],
|
||||||
|
*,
|
||||||
stats: bool = False,
|
stats: bool = False,
|
||||||
versions: bool = False,
|
versions: bool = False,
|
||||||
) -> list[ThingDetails]:
|
) -> list[ThingDetails]:
|
||||||
@@ -155,9 +165,9 @@ class BGGClient:
|
|||||||
def collection(
|
def collection(
|
||||||
self,
|
self,
|
||||||
username: str,
|
username: str,
|
||||||
|
*,
|
||||||
subtype: str | None = None,
|
subtype: str | None = None,
|
||||||
version: bool = True,
|
version: bool = True,
|
||||||
*,
|
|
||||||
refresh: bool = False,
|
refresh: bool = False,
|
||||||
) -> list[CollectionItem]:
|
) -> list[CollectionItem]:
|
||||||
params = {"username": username, "own": "1"}
|
params = {"username": username, "own": "1"}
|
||||||
@@ -179,3 +189,8 @@ class BGGClient:
|
|||||||
)
|
)
|
||||||
seen = {item.coll_id for item in base}
|
seen = {item.coll_id for item in base}
|
||||||
return base + [e for e in expansions if e.coll_id not in seen]
|
return base + [e for e in expansions if e.coll_id not in seen]
|
||||||
|
|
||||||
|
|
||||||
|
def client_for(cfg: Config) -> BGGClient:
|
||||||
|
"""The standard injection fallback: every stage's `client or client_for(cfg)`."""
|
||||||
|
return BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
||||||
|
|||||||
+8
-3
@@ -7,7 +7,7 @@ from typing import Annotated
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from bggpipe.config import load_config
|
from bggpipe.config import DEFAULT_REVIEW_PORT, load_config
|
||||||
|
|
||||||
app = typer.Typer(
|
app = typer.Typer(
|
||||||
help="Shelf-to-BGG collection pipeline.",
|
help="Shelf-to-BGG collection pipeline.",
|
||||||
@@ -56,7 +56,9 @@ def review(
|
|||||||
web: Annotated[
|
web: Annotated[
|
||||||
bool, typer.Option("--web", help="Serve the review UI on localhost")
|
bool, typer.Option("--web", help="Serve the review UI on localhost")
|
||||||
] = False,
|
] = False,
|
||||||
port: Annotated[int, typer.Option("--port", help="Port for --web")] = 8377,
|
port: Annotated[
|
||||||
|
int, typer.Option("--port", help="Port for --web")
|
||||||
|
] = DEFAULT_REVIEW_PORT,
|
||||||
dev: Annotated[
|
dev: Annotated[
|
||||||
bool, typer.Option("--dev", help="With --web: restart on source changes")
|
bool, typer.Option("--dev", help="With --web: restart on source changes")
|
||||||
] = False,
|
] = False,
|
||||||
@@ -122,7 +124,10 @@ def upload(
|
|||||||
|
|
||||||
@app.command()
|
@app.command()
|
||||||
def enrich(
|
def enrich(
|
||||||
refresh: Annotated[bool, typer.Option("--refresh")] = False,
|
refresh: Annotated[
|
||||||
|
bool,
|
||||||
|
typer.Option("--refresh", help="Re-fetch metadata (ranks/ratings drift)"),
|
||||||
|
] = False,
|
||||||
config: ConfigOpt = None,
|
config: ConfigOpt = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Stage 6: fetch full game + version metadata into games.json."""
|
"""Stage 6: fetch full game + version metadata into games.json."""
|
||||||
|
|||||||
@@ -10,10 +10,12 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import os
|
import os
|
||||||
import tomllib
|
import tomllib
|
||||||
|
import warnings
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
DEFAULT_CONFIG_PATH = Path("config.toml")
|
DEFAULT_CONFIG_PATH = Path("config.toml")
|
||||||
|
DEFAULT_REVIEW_PORT = 8377
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -40,6 +42,38 @@ class Config:
|
|||||||
def matches_path(self) -> Path:
|
def matches_path(self) -> Path:
|
||||||
return self.data_dir / "matches.csv"
|
return self.data_dir / "matches.csv"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def extract_raw_dir(self) -> Path:
|
||||||
|
return self.data_dir / "extract_raw"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def to_add_path(self) -> Path:
|
||||||
|
return self.data_dir / "to_add.csv"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def to_update_path(self) -> Path:
|
||||||
|
return self.data_dir / "to_update.csv"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def upload_log_path(self) -> Path:
|
||||||
|
return self.data_dir / "upload_log.csv"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def games_path(self) -> Path:
|
||||||
|
return self.data_dir / "games.json"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def dismissed_path(self) -> Path:
|
||||||
|
return self.data_dir / "unidentified_dismissed.json"
|
||||||
|
|
||||||
|
@property
|
||||||
|
def stub_marker_paths(self) -> tuple[Path, Path]:
|
||||||
|
# gitignored (travels with the stub XML) + committed (guards clones)
|
||||||
|
return (
|
||||||
|
self.cache_dir / "STUB_FIXTURES.marker",
|
||||||
|
self.data_dir / "STUB_DATA.marker",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def load_config(path: Path | None = None) -> Config:
|
def load_config(path: Path | None = None) -> Config:
|
||||||
cfg = Config()
|
cfg = Config()
|
||||||
@@ -53,6 +87,13 @@ def load_config(path: Path | None = None) -> Config:
|
|||||||
"rate_limit_seconds": float,
|
"rate_limit_seconds": float,
|
||||||
}
|
}
|
||||||
updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw}
|
updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw}
|
||||||
|
if unknown := sorted(raw.keys() - known.keys()):
|
||||||
|
# a typo'd key silently falling back to defaults is a debugging
|
||||||
|
# trap ("No photos found in photos/") — say so up front
|
||||||
|
warnings.warn(
|
||||||
|
f"{p}: ignoring unknown key(s): {', '.join(unknown)}",
|
||||||
|
stacklevel=2,
|
||||||
|
)
|
||||||
cfg = replace(cfg, **updates)
|
cfg = replace(cfg, **updates)
|
||||||
if username := os.environ.get("BGG_USERNAME"):
|
if username := os.environ.get("BGG_USERNAME"):
|
||||||
cfg = replace(cfg, bgg_username=username)
|
cfg = replace(cfg, bgg_username=username)
|
||||||
|
|||||||
+38
-26
@@ -8,11 +8,12 @@ Two collection sources:
|
|||||||
logged-in-user exemption.
|
logged-in-user exemption.
|
||||||
|
|
||||||
Outputs both artifacts:
|
Outputs both artifacts:
|
||||||
- to_add.csv — recognized games not in the collection (new entries);
|
- to_add.csv — recognized games not in the collection, including
|
||||||
|
additional copies whose confident version matches no owned copy;
|
||||||
- to_update.csv — owned, VERSION-LESS entries where matching produced a
|
- to_update.csv — owned, VERSION-LESS entries where matching produced a
|
||||||
confident version (version_auto/version_approved). Strictly additive:
|
confident version (version_auto/version_approved). Strictly additive:
|
||||||
entries that already carry a version are never touched — a photo/version
|
entries that already carry a version are never touched — a further copy
|
||||||
disagreement is reported in the summary instead.
|
with a different version becomes a to_add row instead.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -24,9 +25,13 @@ from pathlib import Path
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGClient
|
from bggpipe.bgg_client import BGGClient, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.models import CollectionItem, parse_collection
|
from bggpipe.models import (
|
||||||
|
CONFIDENT_VERSION_STATUSES,
|
||||||
|
CollectionItem,
|
||||||
|
parse_collection,
|
||||||
|
)
|
||||||
from bggpipe.resolve import read_matches
|
from bggpipe.resolve import read_matches
|
||||||
|
|
||||||
TO_ADD_COLUMNS = [
|
TO_ADD_COLUMNS = [
|
||||||
@@ -42,7 +47,6 @@ TO_ADD_COLUMNS = [
|
|||||||
TO_UPDATE_COLUMNS = ["collid", "bgg_id", "bgg_name", "version_id", "version_name"]
|
TO_UPDATE_COLUMNS = ["collid", "bgg_id", "bgg_name", "version_id", "version_name"]
|
||||||
|
|
||||||
SNAPSHOT_FILES = ("collection_snapshot_base.xml", "collection_snapshot_expansions.xml")
|
SNAPSHOT_FILES = ("collection_snapshot_base.xml", "collection_snapshot_expansions.xml")
|
||||||
_CONFIDENT_VERSION = ("version_auto", "version_approved")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -50,7 +54,7 @@ class DiffResult:
|
|||||||
to_add: list[dict] = field(default_factory=list)
|
to_add: list[dict] = field(default_factory=list)
|
||||||
to_update: list[dict] = field(default_factory=list)
|
to_update: list[dict] = field(default_factory=list)
|
||||||
already_owned: list[str] = field(default_factory=list) # title_raw
|
already_owned: list[str] = field(default_factory=list) # title_raw
|
||||||
disagreements: list[str] = field(default_factory=list)
|
second_copies: list[str] = field(default_factory=list) # notes for adds
|
||||||
unseen: list[CollectionItem] = field(default_factory=list)
|
unseen: list[CollectionItem] = field(default_factory=list)
|
||||||
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
|
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
|
||||||
rejected: int = 0
|
rejected: int = 0
|
||||||
@@ -113,14 +117,15 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
if copies:
|
if copies:
|
||||||
seen_object_ids.add(bgg_id)
|
seen_object_ids.add(bgg_id)
|
||||||
|
|
||||||
confident = row["version_status"] in _CONFIDENT_VERSION and row["version_id"]
|
confident = (
|
||||||
|
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
|
||||||
|
)
|
||||||
version_id = int(row["version_id"]) if confident else None
|
version_id = int(row["version_id"]) if confident else None
|
||||||
|
|
||||||
if not copies:
|
def add_row(row: dict = row, confident: bool = confident) -> dict:
|
||||||
photos = {p for p in row["source_photos"].split(";") if p}
|
photos = {p for p in row["source_photos"].split(";") if p}
|
||||||
photos |= merged_photos.get(row["title_raw"], set())
|
photos |= merged_photos.get(row["title_raw"], set())
|
||||||
result.to_add.append(
|
return {
|
||||||
{
|
|
||||||
"bgg_id": row["bgg_id"],
|
"bgg_id": row["bgg_id"],
|
||||||
"bgg_name": row["bgg_name"],
|
"bgg_name": row["bgg_name"],
|
||||||
"year": row["year"],
|
"year": row["year"],
|
||||||
@@ -130,7 +135,9 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
"title_raw": row["title_raw"],
|
"title_raw": row["title_raw"],
|
||||||
"source_photos": ";".join(sorted(photos)),
|
"source_photos": ";".join(sorted(photos)),
|
||||||
}
|
}
|
||||||
)
|
|
||||||
|
if not copies:
|
||||||
|
result.to_add.append(add_row())
|
||||||
continue
|
continue
|
||||||
|
|
||||||
if not confident:
|
if not confident:
|
||||||
@@ -161,12 +168,15 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
# every copy already carries a (different) version — never touch it
|
# no remaining copy can take this version: every copy either
|
||||||
result.already_owned.append(row["title_raw"])
|
# already carries a different version or was consumed by another
|
||||||
result.disagreements.append(
|
# match — per spec this is an additional physical copy to ADD
|
||||||
f"{row['title_raw']}: photo suggests version "
|
# (existing entries are never touched)
|
||||||
f"{row['version_name']!r} ({row['version_id']}) but the "
|
result.to_add.append(add_row())
|
||||||
"collection entry already has a version set — not changing it"
|
result.second_copies.append(
|
||||||
|
f"{row['title_raw']}: adding as a NEW copy with version "
|
||||||
|
f"{row['version_name']!r} ({row['version_id']}) — every "
|
||||||
|
"existing entry of this game keeps its current version"
|
||||||
)
|
)
|
||||||
|
|
||||||
result.unseen = [
|
result.unseen = [
|
||||||
@@ -177,10 +187,12 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
|
|
||||||
def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None:
|
def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
with path.open("w", newline="") as f:
|
tmp = path.with_name(path.name + ".tmp")
|
||||||
|
with tmp.open("w", newline="") as f:
|
||||||
writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore")
|
writer = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore")
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
writer.writerows(rows)
|
writer.writerows(rows)
|
||||||
|
os.replace(tmp, path) # atomic: a killed diff never leaves a torn queue
|
||||||
|
|
||||||
|
|
||||||
def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
||||||
@@ -191,8 +203,8 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
|||||||
|
|
||||||
if os.environ.get("BGG_API_TOKEN") and cfg.bgg_username:
|
if os.environ.get("BGG_API_TOKEN") and cfg.bgg_username:
|
||||||
typer.echo("Fetching live collection from BGG…")
|
typer.echo("Fetching live collection from BGG…")
|
||||||
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
client = client or client_for(cfg)
|
||||||
collection = client.collection_full(cfg.bgg_username)
|
collection = client.collection_full(cfg.bgg_username, refresh=True)
|
||||||
else:
|
else:
|
||||||
typer.echo(
|
typer.echo(
|
||||||
"No BGG_API_TOKEN — using collection snapshot files in "
|
"No BGG_API_TOKEN — using collection snapshot files in "
|
||||||
@@ -202,8 +214,8 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
|||||||
|
|
||||||
result = compute_diff(rows, collection)
|
result = compute_diff(rows, collection)
|
||||||
|
|
||||||
_write_csv(cfg.data_dir / "to_add.csv", TO_ADD_COLUMNS, result.to_add)
|
_write_csv(cfg.to_add_path, TO_ADD_COLUMNS, result.to_add)
|
||||||
_write_csv(cfg.data_dir / "to_update.csv", TO_UPDATE_COLUMNS, result.to_update)
|
_write_csv(cfg.to_update_path, TO_UPDATE_COLUMNS, result.to_update)
|
||||||
|
|
||||||
merged_note = f" · {result.merged} merged duplicate(s)" if result.merged else ""
|
merged_note = f" · {result.merged} merged duplicate(s)" if result.merged else ""
|
||||||
typer.echo(
|
typer.echo(
|
||||||
@@ -212,9 +224,9 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
|||||||
f"update(s) · {len(result.pending)} pending review · "
|
f"update(s) · {len(result.pending)} pending review · "
|
||||||
f"{result.rejected} rejected{merged_note}"
|
f"{result.rejected} rejected{merged_note}"
|
||||||
)
|
)
|
||||||
if result.disagreements:
|
if result.second_copies:
|
||||||
typer.echo("\nVersion disagreements (left untouched):")
|
typer.echo("\nSecond copies (verify these on the dry run before upload):")
|
||||||
for line in result.disagreements:
|
for line in result.second_copies:
|
||||||
typer.echo(f" - {line}")
|
typer.echo(f" - {line}")
|
||||||
if result.pending:
|
if result.pending:
|
||||||
typer.echo("\nStill pending review: " + ", ".join(result.pending))
|
typer.echo("\nStill pending review: " + ", ".join(result.pending))
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ def run_enrich(
|
|||||||
key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
|
key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
|
||||||
targets.append((key, int(row["bgg_id"]), version))
|
targets.append((key, int(row["bgg_id"]), version))
|
||||||
|
|
||||||
games_path = cfg.data_dir / "games.json"
|
games_path = cfg.games_path
|
||||||
games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
|
games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
|
||||||
|
|
||||||
need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
|
need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
|
||||||
|
|||||||
+17
-7
@@ -103,7 +103,7 @@ def prepare_image(path: Path) -> tuple[str, str]:
|
|||||||
return base64.standard_b64encode(buffer.getvalue()).decode(), "image/jpeg"
|
return base64.standard_b64encode(buffer.getvalue()).decode(), "image/jpeg"
|
||||||
|
|
||||||
|
|
||||||
def parse_vision_response(text: str) -> tuple[list[dict], list[dict]]:
|
def parse_vision_response(text: str) -> tuple[list[dict], list[dict], int]:
|
||||||
"""Parse the model's JSON defensively: strip code fences, locate the
|
"""Parse the model's JSON defensively: strip code fences, locate the
|
||||||
payload amid any prose. Returns (title entries, unidentified sightings).
|
payload amid any prose. Returns (title entries, unidentified sightings).
|
||||||
A bare JSON array (the pre-unidentified response shape) still parses —
|
A bare JSON array (the pre-unidentified response shape) still parses —
|
||||||
@@ -133,7 +133,8 @@ def parse_vision_response(text: str) -> tuple[list[dict], list[dict]]:
|
|||||||
if isinstance(u, dict)
|
if isinstance(u, dict)
|
||||||
and (u.get("location") or u.get("partial_text") or u.get("art_notes"))
|
and (u.get("location") or u.get("partial_text") or u.get("art_notes"))
|
||||||
]
|
]
|
||||||
return titles, unidentified
|
dropped = len(titles_raw) - len(titles)
|
||||||
|
return titles, unidentified, dropped
|
||||||
|
|
||||||
|
|
||||||
def default_vision(model: str) -> VisionFn:
|
def default_vision(model: str) -> VisionFn:
|
||||||
@@ -172,7 +173,9 @@ def extract_photo(photo: Path, vision: VisionFn) -> dict:
|
|||||||
"""One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache
|
"""One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache
|
||||||
file format)."""
|
file format)."""
|
||||||
image_b64, media_type = prepare_image(photo)
|
image_b64, media_type = prepare_image(photo)
|
||||||
raw_entries, raw_unidentified = parse_vision_response(vision(image_b64, media_type))
|
raw_entries, raw_unidentified, dropped = parse_vision_response(
|
||||||
|
vision(image_b64, media_type)
|
||||||
|
)
|
||||||
unidentified = [
|
unidentified = [
|
||||||
{
|
{
|
||||||
"location": str(u.get("location") or "").strip(),
|
"location": str(u.get("location") or "").strip(),
|
||||||
@@ -198,10 +201,10 @@ def extract_photo(photo: Path, vision: VisionFn) -> dict:
|
|||||||
"source_photos": [photo.name],
|
"source_photos": [photo.name],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
return {"titles": entries, "unidentified": unidentified}
|
return {"titles": entries, "unidentified": unidentified, "dropped": dropped}
|
||||||
|
|
||||||
|
|
||||||
def _cues_conflict(a: dict, b: dict) -> bool:
|
def cues_conflict(a: dict, b: dict) -> bool:
|
||||||
"""Two sightings conflict if any edition cue is set on both and differs —
|
"""Two sightings conflict if any edition cue is set on both and differs —
|
||||||
that means visibly different boxes, i.e. separate editions (spec)."""
|
that means visibly different boxes, i.e. separate editions (spec)."""
|
||||||
for key in ("publisher_hint", "edition_hint", "language_hint"):
|
for key in ("publisher_hint", "edition_hint", "language_hint"):
|
||||||
@@ -238,7 +241,7 @@ def dedupe_entries(entries: list[dict]) -> list[dict]:
|
|||||||
for existing in result:
|
for existing in result:
|
||||||
if existing["title_normalized"] == entry[
|
if existing["title_normalized"] == entry[
|
||||||
"title_normalized"
|
"title_normalized"
|
||||||
] and not _cues_conflict(existing, entry):
|
] and not cues_conflict(existing, entry):
|
||||||
existing.update(_merge(existing, entry))
|
existing.update(_merge(existing, entry))
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -298,7 +301,7 @@ def run_extract(
|
|||||||
)
|
)
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
raw_dir = cfg.data_dir / "extract_raw"
|
raw_dir = cfg.extract_raw_dir
|
||||||
raw_dir.mkdir(parents=True, exist_ok=True)
|
raw_dir.mkdir(parents=True, exist_ok=True)
|
||||||
vision = vision or default_vision(cfg.model)
|
vision = vision or default_vision(cfg.model)
|
||||||
|
|
||||||
@@ -314,6 +317,13 @@ def run_extract(
|
|||||||
if result["unidentified"]
|
if result["unidentified"]
|
||||||
else ""
|
else ""
|
||||||
)
|
)
|
||||||
|
if result.get("dropped"):
|
||||||
|
# the prompt forbids silently omitting a box; so do we
|
||||||
|
note += (
|
||||||
|
f" — DROPPED {result['dropped']} malformed entr"
|
||||||
|
f"{'y' if result['dropped'] == 1 else 'ies'}; inspect "
|
||||||
|
f"{raw_path}"
|
||||||
|
)
|
||||||
typer.echo(f" {photo.name}: {len(result['titles'])} title(s){note}")
|
typer.echo(f" {photo.name}: {len(result['titles'])} title(s){note}")
|
||||||
|
|
||||||
deduped, unidentified = rebuild_artifacts(
|
deduped, unidentified = rebuild_artifacts(
|
||||||
|
|||||||
+20
-2
@@ -12,6 +12,12 @@ class BGGResponseError(Exception):
|
|||||||
"""The API returned a well-formed error document (e.g. bad username)."""
|
"""The API returned a well-formed error document (e.g. bad username)."""
|
||||||
|
|
||||||
|
|
||||||
|
# The one status predicate the whole pipeline shares: a version is trusted
|
||||||
|
# for diff/upload/enrich only when matching produced it confidently or a
|
||||||
|
# human approved it.
|
||||||
|
CONFIDENT_VERSION_STATUSES = ("version_auto", "version_approved")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class SearchResult:
|
class SearchResult:
|
||||||
bgg_id: int
|
bgg_id: int
|
||||||
@@ -209,6 +215,16 @@ def parse_things_full(xml_text: str) -> list[dict]:
|
|||||||
return games
|
return games
|
||||||
|
|
||||||
|
|
||||||
|
def _required_attr(item, name: str) -> str:
|
||||||
|
value = item.get(name)
|
||||||
|
if not value:
|
||||||
|
raise BGGResponseError(
|
||||||
|
f"collection item missing {name!r} — truncated or unexpected "
|
||||||
|
"response; refusing to feed it to the diff"
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
def parse_collection(xml_text: str) -> list[CollectionItem]:
|
def parse_collection(xml_text: str) -> list[CollectionItem]:
|
||||||
items = []
|
items = []
|
||||||
for item in _root(xml_text).findall("item"):
|
for item in _root(xml_text).findall("item"):
|
||||||
@@ -217,8 +233,10 @@ def parse_collection(xml_text: str) -> list[CollectionItem]:
|
|||||||
version_item = item.find("version/item")
|
version_item = item.find("version/item")
|
||||||
items.append(
|
items.append(
|
||||||
CollectionItem(
|
CollectionItem(
|
||||||
object_id=int(item.get("objectid", 0)),
|
# strict: a missing id coerced to 0 would collide in the
|
||||||
coll_id=int(item.get("collid", 0)),
|
# collid dedupe and silently drop owned games from the diff
|
||||||
|
object_id=int(_required_attr(item, "objectid")),
|
||||||
|
coll_id=int(_required_attr(item, "collid")),
|
||||||
name=item.findtext("name", default=""),
|
name=item.findtext("name", default=""),
|
||||||
subtype=item.get("subtype", "boardgame"),
|
subtype=item.get("subtype", "boardgame"),
|
||||||
own=status is not None and status.get("own") == "1",
|
own=status is not None and status.get("own") == "1",
|
||||||
|
|||||||
+70
-47
@@ -12,16 +12,17 @@ import csv
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
from collections import Counter
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import typer
|
import typer
|
||||||
from rapidfuzz import fuzz
|
from rapidfuzz import fuzz
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGAuthError, BGGClient
|
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.extract import _cues_conflict
|
from bggpipe.extract import cues_conflict
|
||||||
from bggpipe.models import GameVersion
|
from bggpipe.models import CONFIDENT_VERSION_STATUSES, GameVersion
|
||||||
from bggpipe.normalize import normalize_title
|
from bggpipe.normalize import normalize_title
|
||||||
|
|
||||||
FUZZY_THRESHOLD = 90
|
FUZZY_THRESHOLD = 90
|
||||||
@@ -30,6 +31,10 @@ FUZZY_THRESHOLD = 90
|
|||||||
DOMINANCE_MIN_OWNED = 100
|
DOMINANCE_MIN_OWNED = 100
|
||||||
DOMINANCE_FACTOR = 10
|
DOMINANCE_FACTOR = 10
|
||||||
VERSION_PLAUSIBLE_SCORE = 2
|
VERSION_PLAUSIBLE_SCORE = 2
|
||||||
|
# "Is this the publisher on the box?" — one answer, asked in two places
|
||||||
|
# (candidate tie-break and version scoring): the sites must move together.
|
||||||
|
PUBLISHER_MATCH_THRESHOLD = 85
|
||||||
|
EDITION_NAME_THRESHOLD = 80
|
||||||
|
|
||||||
MATCH_COLUMNS = [
|
MATCH_COLUMNS = [
|
||||||
"title_raw",
|
"title_raw",
|
||||||
@@ -45,6 +50,7 @@ MATCH_COLUMNS = [
|
|||||||
"version_candidates_json",
|
"version_candidates_json",
|
||||||
"source_photos",
|
"source_photos",
|
||||||
"merged_into",
|
"merged_into",
|
||||||
|
"dedupe_veto",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -127,6 +133,8 @@ class MatchRow:
|
|||||||
self.version_candidates, ensure_ascii=False
|
self.version_candidates, ensure_ascii=False
|
||||||
),
|
),
|
||||||
"source_photos": ";".join(self.source_photos),
|
"source_photos": ";".join(self.source_photos),
|
||||||
|
"merged_into": "",
|
||||||
|
"dedupe_veto": "",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -198,6 +206,10 @@ def _plausible_candidates(
|
|||||||
by_id: dict[int, Candidate] = {}
|
by_id: dict[int, Candidate] = {}
|
||||||
for result in client.search(query):
|
for result in client.search(query):
|
||||||
norm = normalize_title(result.name)
|
norm = normalize_title(result.name)
|
||||||
|
# a fully non-Latin title normalizes to "" — empty-vs-empty is not a
|
||||||
|
# match (and token_sort_ratio("", "") is 100, so guard fuzzy too)
|
||||||
|
if not entry.title_normalized:
|
||||||
|
continue
|
||||||
exact = norm == entry.title_normalized
|
exact = norm == entry.title_normalized
|
||||||
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
|
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
|
||||||
if not exact and fuzzy < FUZZY_THRESHOLD:
|
if not exact and fuzzy < FUZZY_THRESHOLD:
|
||||||
@@ -259,13 +271,19 @@ def _publisher_pick(entry: TitleEntry, top: list[Candidate]) -> Candidate | None
|
|||||||
candidate is from that publisher, that's the game."""
|
candidate is from that publisher, that's the game."""
|
||||||
if not entry.publisher_hint:
|
if not entry.publisher_hint:
|
||||||
return None
|
return None
|
||||||
|
if len({c.type for c in top}) > 1:
|
||||||
|
# mixed base-game/expansion candidates are never auto-resolved —
|
||||||
|
# the same veto _dominant applies (spec's top failure mode); an
|
||||||
|
# alternate name can make an expansion "exact" too
|
||||||
|
return None
|
||||||
hint = normalize_title(entry.publisher_hint)
|
hint = normalize_title(entry.publisher_hint)
|
||||||
matches = [
|
matches = [
|
||||||
c
|
c
|
||||||
for c in top
|
for c in top
|
||||||
if c.exact
|
if c.exact
|
||||||
and any(
|
and any(
|
||||||
fuzz.partial_ratio(hint, normalize_title(p)) >= 85 for p in c.publishers
|
fuzz.partial_ratio(hint, normalize_title(p)) >= PUBLISHER_MATCH_THRESHOLD
|
||||||
|
for p in c.publishers
|
||||||
)
|
)
|
||||||
]
|
]
|
||||||
return matches[0] if len(matches) == 1 else None
|
return matches[0] if len(matches) == 1 else None
|
||||||
@@ -296,7 +314,7 @@ def _score_version(entry: TitleEntry, version: GameVersion) -> int:
|
|||||||
if entry.publisher_hint:
|
if entry.publisher_hint:
|
||||||
hint = normalize_title(entry.publisher_hint)
|
hint = normalize_title(entry.publisher_hint)
|
||||||
if any(
|
if any(
|
||||||
fuzz.partial_ratio(hint, normalize_title(p)) >= 85
|
fuzz.partial_ratio(hint, normalize_title(p)) >= PUBLISHER_MATCH_THRESHOLD
|
||||||
for p in version.publishers
|
for p in version.publishers
|
||||||
):
|
):
|
||||||
score += 2
|
score += 2
|
||||||
@@ -312,18 +330,24 @@ def _score_version(entry: TitleEntry, version: GameVersion) -> int:
|
|||||||
and fuzz.token_set_ratio(
|
and fuzz.token_set_ratio(
|
||||||
normalize_title(entry.edition_hint), normalize_title(version.name)
|
normalize_title(entry.edition_hint), normalize_title(version.name)
|
||||||
)
|
)
|
||||||
>= 80
|
>= EDITION_NAME_THRESHOLD
|
||||||
):
|
):
|
||||||
score += 2
|
score += 2
|
||||||
return score
|
return score
|
||||||
|
|
||||||
|
|
||||||
def _resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None:
|
def resolve_version(client: BGGClient, entry: TitleEntry, row: MatchRow) -> None:
|
||||||
"""Fill version_* fields on an auto/approved row. Never guess (spec)."""
|
"""Fill version_* fields on an auto/approved row. Never guess (spec).
|
||||||
|
Public: review's manual-id path calls this too."""
|
||||||
if not entry.has_version_cues:
|
if not entry.has_version_cues:
|
||||||
row.version_status = "version_unknown"
|
row.version_status = "version_unknown"
|
||||||
return
|
return
|
||||||
(thing,) = client.things([row.bgg_id], versions=True)
|
things = client.things([row.bgg_id], versions=True)
|
||||||
|
if not things:
|
||||||
|
# a manually-typed id BGG doesn't know: nothing to offer
|
||||||
|
row.version_status = "version_unknown"
|
||||||
|
return
|
||||||
|
thing = things[0]
|
||||||
scored = sorted(
|
scored = sorted(
|
||||||
((v, _score_version(entry, v)) for v in thing.versions),
|
((v, _score_version(entry, v)) for v in thing.versions),
|
||||||
key=lambda pair: -pair[1],
|
key=lambda pair: -pair[1],
|
||||||
@@ -364,14 +388,10 @@ def resolve_entry(client: BGGClient, entry: TitleEntry) -> MatchRow:
|
|||||||
break
|
break
|
||||||
row = _classify(client, entry, cands)
|
row = _classify(client, entry, cands)
|
||||||
if row.match_status == "auto":
|
if row.match_status == "auto":
|
||||||
_resolve_version(client, entry, row)
|
resolve_version(client, entry, row)
|
||||||
return row
|
return row
|
||||||
|
|
||||||
|
|
||||||
def _row_key(title_raw: str, source_photos: str) -> tuple[str, str]:
|
|
||||||
return (title_raw, source_photos)
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class MergeEvent:
|
class MergeEvent:
|
||||||
loser_title: str
|
loser_title: str
|
||||||
@@ -409,9 +429,13 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
|
|||||||
for row in rows:
|
for row in rows:
|
||||||
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
|
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
|
||||||
continue
|
continue
|
||||||
|
if row.get("dedupe_veto"):
|
||||||
|
# a human already ruled "this is a genuinely separate copy" —
|
||||||
|
# re-running resolve must never overturn that (spec: re-runs
|
||||||
|
# lose no work, least of all review decisions)
|
||||||
|
continue
|
||||||
confident = (
|
confident = (
|
||||||
row["version_status"] in ("version_auto", "version_approved")
|
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
|
||||||
and row["version_id"]
|
|
||||||
)
|
)
|
||||||
key = (row["bgg_id"], row["version_id"] if confident else "")
|
key = (row["bgg_id"], row["version_id"] if confident else "")
|
||||||
groups.setdefault(key, []).append(row)
|
groups.setdefault(key, []).append(row)
|
||||||
@@ -429,7 +453,7 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
|
|||||||
)
|
)
|
||||||
survivor = group[0]
|
survivor = group[0]
|
||||||
for loser in group[1:]:
|
for loser in group[1:]:
|
||||||
if _cues_conflict(cues(survivor), cues(loser)):
|
if cues_conflict(cues(survivor), cues(loser)):
|
||||||
continue # conflicting edition cues: genuinely two copies
|
continue # conflicting edition cues: genuinely two copies
|
||||||
loser["match_status"] = "merged"
|
loser["match_status"] = "merged"
|
||||||
loser["merged_into"] = survivor["title_raw"]
|
loser["merged_into"] = survivor["title_raw"]
|
||||||
@@ -449,8 +473,9 @@ def read_matches(path: Path) -> list[dict[str, str]]:
|
|||||||
return []
|
return []
|
||||||
with path.open(newline="") as f:
|
with path.open(newline="") as f:
|
||||||
rows = list(csv.DictReader(f))
|
rows = list(csv.DictReader(f))
|
||||||
for row in rows: # files written before the merged_into column existed
|
for row in rows: # files written before these columns existed
|
||||||
row.setdefault("merged_into", "")
|
row.setdefault("merged_into", "")
|
||||||
|
row.setdefault("dedupe_veto", "")
|
||||||
return rows
|
return rows
|
||||||
|
|
||||||
|
|
||||||
@@ -467,39 +492,40 @@ def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
|
|||||||
os.replace(tmp, path)
|
os.replace(tmp, path)
|
||||||
|
|
||||||
|
|
||||||
def read_existing_keys(path: Path) -> set[tuple[str, str]]:
|
|
||||||
if not path.exists():
|
|
||||||
return set()
|
|
||||||
with path.open(newline="") as f:
|
|
||||||
return {_row_key(r["title_raw"], r["source_photos"]) for r in csv.DictReader(f)}
|
|
||||||
|
|
||||||
|
|
||||||
def append_rows(path: Path, rows: list[MatchRow]) -> None:
|
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
is_new = not path.exists()
|
|
||||||
with path.open("a", newline="") as f:
|
|
||||||
writer = csv.DictWriter(f, fieldnames=MATCH_COLUMNS)
|
|
||||||
if is_new:
|
|
||||||
writer.writeheader()
|
|
||||||
for row in rows:
|
|
||||||
writer.writerow(row.to_csv())
|
|
||||||
|
|
||||||
|
|
||||||
def run_resolve(
|
def run_resolve(
|
||||||
cfg: Config, *, force: bool = False, client: BGGClient | None = None
|
cfg: Config, *, force: bool = False, client: BGGClient | None = None
|
||||||
) -> list[MatchRow]:
|
) -> list[MatchRow]:
|
||||||
entries = load_titles(cfg.titles_path)
|
entries = load_titles(cfg.titles_path)
|
||||||
if force and cfg.matches_path.exists():
|
if force and cfg.matches_path.exists():
|
||||||
cfg.matches_path.unlink()
|
cfg.matches_path.unlink()
|
||||||
existing = read_existing_keys(cfg.matches_path)
|
existing_rows = read_matches(cfg.matches_path)
|
||||||
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
client = client or client_for(cfg)
|
||||||
|
|
||||||
|
# Pair entries with existing rows BY TITLE, positionally, not by exact
|
||||||
|
# (title, photos) key: extract unions a new photo of an already-resolved
|
||||||
|
# game into its entry, and that must update the row's provenance — not
|
||||||
|
# re-resolve the game as a duplicate row. Same-title entries only stay
|
||||||
|
# separate when their cues conflict (two editions), and those pair up
|
||||||
|
# in stable file order on both sides.
|
||||||
|
rows_by_title: dict[str, list[dict]] = {}
|
||||||
|
for row in existing_rows:
|
||||||
|
rows_by_title.setdefault(row["title_raw"], []).append(row)
|
||||||
|
seen_per_title: dict[str, int] = {}
|
||||||
|
|
||||||
new_rows: list[MatchRow] = []
|
new_rows: list[MatchRow] = []
|
||||||
skipped = 0
|
skipped = 0
|
||||||
|
photos_updated = False
|
||||||
blocked: list[str] = []
|
blocked: list[str] = []
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
key = _row_key(entry.title_raw, ";".join(entry.source_photos))
|
ix = seen_per_title.get(entry.title_raw, 0)
|
||||||
if key in existing:
|
seen_per_title[entry.title_raw] = ix + 1
|
||||||
|
paired = rows_by_title.get(entry.title_raw, [])
|
||||||
|
if ix < len(paired):
|
||||||
|
row_dict = paired[ix]
|
||||||
|
photos = ";".join(entry.source_photos)
|
||||||
|
if row_dict["source_photos"] != photos:
|
||||||
|
row_dict["source_photos"] = photos
|
||||||
|
photos_updated = True
|
||||||
skipped += 1
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
@@ -515,12 +541,11 @@ def run_resolve(
|
|||||||
version = f" [{row.version_status}]" if row.version_status else ""
|
version = f" [{row.version_status}]" if row.version_status else ""
|
||||||
typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}")
|
typer.echo(f" {entry.title_raw!r} -> {row.match_status}: {detail}{version}")
|
||||||
|
|
||||||
append_rows(cfg.matches_path, new_rows)
|
all_rows = existing_rows + [row.to_csv() for row in new_rows]
|
||||||
|
|
||||||
all_rows = read_matches(cfg.matches_path)
|
|
||||||
merges = dedupe_matches(all_rows, entries)
|
merges = dedupe_matches(all_rows, entries)
|
||||||
|
if new_rows or photos_updated or merges:
|
||||||
|
write_matches(cfg.matches_path, all_rows) # atomic full rewrite
|
||||||
if merges:
|
if merges:
|
||||||
write_matches(cfg.matches_path, all_rows)
|
|
||||||
typer.echo("")
|
typer.echo("")
|
||||||
for m in merges:
|
for m in merges:
|
||||||
typer.echo(
|
typer.echo(
|
||||||
@@ -528,9 +553,7 @@ def run_resolve(
|
|||||||
f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong"
|
f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong"
|
||||||
)
|
)
|
||||||
|
|
||||||
counts: dict[str, int] = {}
|
counts = Counter(row.match_status for row in new_rows)
|
||||||
for row in new_rows:
|
|
||||||
counts[row.match_status] = counts.get(row.match_status, 0) + 1
|
|
||||||
summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items()))
|
summary = ", ".join(f"{n} {status}" for status, n in sorted(counts.items()))
|
||||||
typer.echo(
|
typer.echo(
|
||||||
f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); "
|
f"Resolved {len(new_rows)} title(s) ({summary or 'nothing new'}); "
|
||||||
|
|||||||
+74
-29
@@ -1,9 +1,11 @@
|
|||||||
"""Stage 3 — human review of ambiguous/unmatched matches, then versions.
|
"""Stage 3 — the review decision engine, worn by two faces.
|
||||||
|
|
||||||
A plain rich prompt loop (deliberately dependency-light, per spec). Every
|
ReviewSession owns all decision logic and every matches.csv write: the
|
||||||
decision rewrites matches.csv atomically, so quitting mid-review — q,
|
decision API (decide_*/veto_merge/pending_rows/version_rows) serves both
|
||||||
Ctrl-C, or end of input — loses nothing; the next run resumes with
|
this module's rich TUI prompt loop and webreview's FastAPI endpoints, and
|
||||||
whatever is still ambiguous/unmatched. The version pass is optional and
|
reload_if_changed() lets either face follow external rewrites (a resolve
|
||||||
|
run in another terminal). Every decision rewrites matches.csv atomically,
|
||||||
|
so quitting mid-review loses nothing. The version pass is optional and
|
||||||
skippable: version review must never block getting games uploaded.
|
skippable: version review must never block getting games uploaded.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -14,18 +16,20 @@ from collections.abc import Callable
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
import typer
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
from rich.panel import Panel
|
from rich.panel import Panel
|
||||||
from rich.table import Table
|
from rich.table import Table
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGAuthError, BGGClient, BGGQueueTimeout
|
from bggpipe.bgg_client import BGGAuthError, BGGClient, BGGQueueTimeout, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.models import BGGResponseError
|
from bggpipe.models import BGGResponseError
|
||||||
from bggpipe.resolve import (
|
from bggpipe.resolve import (
|
||||||
MatchRow,
|
MatchRow,
|
||||||
_resolve_version,
|
TitleEntry,
|
||||||
load_titles,
|
load_titles,
|
||||||
read_matches,
|
read_matches,
|
||||||
|
resolve_version,
|
||||||
write_matches,
|
write_matches,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -57,8 +61,9 @@ class ReviewSession:
|
|||||||
self.cfg = cfg
|
self.cfg = cfg
|
||||||
self.console = console or Console()
|
self.console = console or Console()
|
||||||
self.input_fn = input_fn or (lambda prompt: self.console.input(prompt))
|
self.input_fn = input_fn or (lambda prompt: self.console.input(prompt))
|
||||||
self.client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
self.client = client or client_for(cfg)
|
||||||
self.decisions = 0
|
self.decisions = 0
|
||||||
|
self.warnings: list[str] = []
|
||||||
self._load()
|
self._load()
|
||||||
|
|
||||||
# -- plumbing -------------------------------------------------------
|
# -- plumbing -------------------------------------------------------
|
||||||
@@ -99,8 +104,34 @@ class ReviewSession:
|
|||||||
raise _Quit
|
raise _Quit
|
||||||
return answer
|
return answer
|
||||||
|
|
||||||
def _save(self) -> None:
|
def _warn(self, message: str) -> None:
|
||||||
|
"""Degradations must be visible in BOTH faces: the console for the
|
||||||
|
TUI, and self.warnings for the web UI (whose console is a StringIO)."""
|
||||||
|
self.warnings.append(message)
|
||||||
|
self.console.print(f"[yellow]{message}[/yellow]")
|
||||||
|
|
||||||
|
def _save(self, row: dict | None = None) -> None:
|
||||||
|
"""Atomic write of the in-memory rows. If another process rewrote the
|
||||||
|
file since we loaded (resolve in a second terminal), reload first and
|
||||||
|
re-apply `row` — the one decision being saved — by identity, so
|
||||||
|
neither side's work is silently lost."""
|
||||||
|
if self.reload_if_changed() and row is not None:
|
||||||
|
key = (row["title_raw"], row["source_photos"])
|
||||||
|
for i, fresh in enumerate(self.rows):
|
||||||
|
if (fresh["title_raw"], fresh["source_photos"]) == key:
|
||||||
|
self.rows[i] = row
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
self._warn(
|
||||||
|
f"{row['title_raw']!r} disappeared from matches.csv while "
|
||||||
|
"you decided — decision NOT saved"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
try:
|
||||||
write_matches(self.cfg.matches_path, self.rows)
|
write_matches(self.cfg.matches_path, self.rows)
|
||||||
|
except OSError:
|
||||||
|
self._load() # memory must never claim what disk doesn't hold
|
||||||
|
raise
|
||||||
self._loaded_mtimes = self._data_mtimes() # own writes aren't "changes"
|
self._loaded_mtimes = self._data_mtimes() # own writes aren't "changes"
|
||||||
self.decisions += 1
|
self.decisions += 1
|
||||||
|
|
||||||
@@ -111,7 +142,7 @@ class ReviewSession:
|
|||||||
row["year"] = str(candidate.get("year") or "")
|
row["year"] = str(candidate.get("year") or "")
|
||||||
row["type"] = candidate.get("type") or "boardgame"
|
row["type"] = candidate.get("type") or "boardgame"
|
||||||
self._fill_version(row)
|
self._fill_version(row)
|
||||||
self._save()
|
self._save(row)
|
||||||
|
|
||||||
def _fill_version(self, row: dict) -> None:
|
def _fill_version(self, row: dict) -> None:
|
||||||
"""Try version resolution for a just-approved row. Degrades gracefully:
|
"""Try version resolution for a just-approved row. Degrades gracefully:
|
||||||
@@ -122,9 +153,9 @@ class ReviewSession:
|
|||||||
return
|
return
|
||||||
shim = MatchRow(title_raw=row["title_raw"], bgg_id=int(row["bgg_id"]))
|
shim = MatchRow(title_raw=row["title_raw"], bgg_id=int(row["bgg_id"]))
|
||||||
try:
|
try:
|
||||||
_resolve_version(self.client, entry, shim)
|
resolve_version(self.client, entry, shim)
|
||||||
except _BGG_ERRORS as err:
|
except _BGG_ERRORS as err:
|
||||||
self.console.print(f"[yellow]version lookup unavailable: {err}[/yellow]")
|
self._warn(f"version lookup unavailable ({err}) — recorded version_unknown")
|
||||||
row["version_status"] = "version_unknown"
|
row["version_status"] = "version_unknown"
|
||||||
return
|
return
|
||||||
row["version_status"] = shim.version_status
|
row["version_status"] = shim.version_status
|
||||||
@@ -155,9 +186,21 @@ class ReviewSession:
|
|||||||
row as a distinct, human-confirmed match."""
|
row as a distinct, human-confirmed match."""
|
||||||
row["match_status"] = "approved"
|
row["match_status"] = "approved"
|
||||||
row["merged_into"] = ""
|
row["merged_into"] = ""
|
||||||
self._save()
|
row["dedupe_veto"] = "1" # persists: resolve re-runs must not re-merge
|
||||||
|
self._save(row)
|
||||||
|
|
||||||
def cues_for(self, title_raw: str):
|
def cues_for(
|
||||||
|
self, title_raw: str, source_photos: str | None = None
|
||||||
|
) -> TitleEntry | None:
|
||||||
|
"""The extraction entry behind a row. Same-title entries (two
|
||||||
|
editions of one game) are told apart by their photo set when given."""
|
||||||
|
if source_photos is not None:
|
||||||
|
for entry in self.titles:
|
||||||
|
if (
|
||||||
|
entry.title_raw == title_raw
|
||||||
|
and ";".join(entry.source_photos) == source_photos
|
||||||
|
):
|
||||||
|
return entry
|
||||||
return self._titles.get(title_raw)
|
return self._titles.get(title_raw)
|
||||||
|
|
||||||
def decide_pick(self, row: dict, candidate: dict) -> None:
|
def decide_pick(self, row: dict, candidate: dict) -> None:
|
||||||
@@ -168,7 +211,7 @@ class ReviewSession:
|
|||||||
|
|
||||||
def decide_reject(self, row: dict) -> None:
|
def decide_reject(self, row: dict) -> None:
|
||||||
row["match_status"] = "rejected"
|
row["match_status"] = "rejected"
|
||||||
self._save()
|
self._save(row)
|
||||||
|
|
||||||
def decide_version(self, row: dict, version_id: int | None) -> None:
|
def decide_version(self, row: dict, version_id: int | None) -> None:
|
||||||
"""Pick a version from the row's stored candidates, or None -> unknown."""
|
"""Pick a version from the row's stored candidates, or None -> unknown."""
|
||||||
@@ -186,7 +229,7 @@ class ReviewSession:
|
|||||||
row["version_status"] = "version_approved"
|
row["version_status"] = "version_approved"
|
||||||
row["version_id"] = str(version_id)
|
row["version_id"] = str(version_id)
|
||||||
row["version_name"] = chosen.get("name") or ""
|
row["version_name"] = chosen.get("name") or ""
|
||||||
self._save()
|
self._save(row)
|
||||||
|
|
||||||
# -- displays -------------------------------------------------------
|
# -- displays -------------------------------------------------------
|
||||||
|
|
||||||
@@ -247,12 +290,19 @@ class ReviewSession:
|
|||||||
def _manual_id(self, row: dict, bgg_id: int) -> None:
|
def _manual_id(self, row: dict, bgg_id: int) -> None:
|
||||||
candidate: dict = {"bgg_id": bgg_id}
|
candidate: dict = {"bgg_id": bgg_id}
|
||||||
try:
|
try:
|
||||||
(thing,) = self.client.things([bgg_id])
|
things = self.client.things([bgg_id])
|
||||||
candidate.update(name=thing.name, year=thing.year, type=thing.type)
|
|
||||||
except _BGG_ERRORS as err:
|
except _BGG_ERRORS as err:
|
||||||
self.console.print(
|
self._warn(
|
||||||
f"[yellow]couldn't look up id {bgg_id} ({err}) — "
|
f"couldn't look up id {bgg_id} ({err}) — recording the id with no name"
|
||||||
"recording the id with no name[/yellow]"
|
)
|
||||||
|
else:
|
||||||
|
if things:
|
||||||
|
thing = things[0]
|
||||||
|
candidate.update(name=thing.name, year=thing.year, type=thing.type)
|
||||||
|
else:
|
||||||
|
self._warn(
|
||||||
|
f"BGG knows no game with id {bgg_id} — recording it "
|
||||||
|
"with no name (typo?)"
|
||||||
)
|
)
|
||||||
self._apply_choice(row, candidate)
|
self._apply_choice(row, candidate)
|
||||||
|
|
||||||
@@ -302,15 +352,10 @@ class ReviewSession:
|
|||||||
# -- entry point ----------------------------------------------------
|
# -- entry point ----------------------------------------------------
|
||||||
|
|
||||||
def run(self) -> None:
|
def run(self) -> None:
|
||||||
pending = [
|
|
||||||
r for r in self.rows if r["match_status"] in ("ambiguous", "unmatched")
|
|
||||||
]
|
|
||||||
try:
|
try:
|
||||||
for row in pending:
|
for row in self.pending_rows():
|
||||||
self._review_match_row(row)
|
self._review_match_row(row)
|
||||||
versions = [
|
versions = self.version_rows()
|
||||||
r for r in self.rows if r["version_status"] == "version_ambiguous"
|
|
||||||
]
|
|
||||||
if versions:
|
if versions:
|
||||||
answer = self._ask(
|
answer = self._ask(
|
||||||
f"{len(versions)} game(s) have ambiguous editions. "
|
f"{len(versions)} game(s) have ambiguous editions. "
|
||||||
@@ -343,6 +388,6 @@ def run_review(
|
|||||||
session.console.print(
|
session.console.print(
|
||||||
f"{cfg.matches_path} is empty — run `bggpipe resolve` first."
|
f"{cfg.matches_path} is empty — run `bggpipe resolve` first."
|
||||||
)
|
)
|
||||||
return session
|
raise typer.Exit(code=1)
|
||||||
session.run()
|
session.run()
|
||||||
return session
|
return session
|
||||||
|
|||||||
@@ -65,6 +65,13 @@
|
|||||||
}
|
}
|
||||||
header kbd { background: rgba(255,255,255,.14); color: var(--paper); border-color: rgba(255,255,255,.25); }
|
header kbd { background: rgba(255,255,255,.14); color: var(--paper); border-color: rgba(255,255,255,.25); }
|
||||||
main { max-width: 62rem; margin: 0 auto; padding: 1.4rem 1.2rem 6rem; }
|
main { max-width: 62rem; margin: 0 auto; padding: 1.4rem 1.2rem 6rem; }
|
||||||
|
#banner { max-width: 62rem; margin: 0 auto; padding: 0 1.2rem; }
|
||||||
|
.banner {
|
||||||
|
border-radius: 6px; padding: .6rem .9rem; margin-top: .9rem;
|
||||||
|
font-size: .85rem; line-height: 1.4;
|
||||||
|
}
|
||||||
|
.banner.error { background: #f6dcd6; border: 1px solid var(--reject); color: #6b2417; }
|
||||||
|
.banner.warn { background: var(--kraft); border: 1px solid var(--brass-deep); color: #6b5b33; }
|
||||||
h2 {
|
h2 {
|
||||||
color: var(--paper);
|
color: var(--paper);
|
||||||
font-family: "Iowan Old Style", Palatino, Georgia, serif;
|
font-family: "Iowan Old Style", Palatino, Georgia, serif;
|
||||||
@@ -236,6 +243,7 @@
|
|||||||
<kbd>v</kbd> veto merge · <kbd>d</kbd> dismiss
|
<kbd>v</kbd> veto merge · <kbd>d</kbd> dismiss
|
||||||
</span>
|
</span>
|
||||||
</header>
|
</header>
|
||||||
|
<div id="banner"></div>
|
||||||
<main id="main"></main>
|
<main id="main"></main>
|
||||||
<script>
|
<script>
|
||||||
"use strict";
|
"use strict";
|
||||||
@@ -245,19 +253,36 @@ let active = 0;
|
|||||||
const esc = s => String(s ?? "").replace(/[&<>"']/g,
|
const esc = s => String(s ?? "").replace(/[&<>"']/g,
|
||||||
c => ({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]));
|
c => ({"&":"&","<":"<",">":">",'"':""","'":"'"}[c]));
|
||||||
|
|
||||||
|
function showBanner(html) {
|
||||||
|
document.getElementById("banner").innerHTML = html;
|
||||||
|
}
|
||||||
|
|
||||||
|
function errorBanner(detail) {
|
||||||
|
showBanner(`<div class="banner error">couldn't load review state: ${esc(detail)}` +
|
||||||
|
` — is the server running? (check its terminal)</div>`);
|
||||||
|
}
|
||||||
|
|
||||||
async function refresh() {
|
async function refresh() {
|
||||||
STATE = await (await fetch("/api/state")).json();
|
const res = await fetch("/api/state");
|
||||||
|
if (!res.ok) throw new Error(`${res.status} ${await res.text().catch(() => res.statusText)}`);
|
||||||
|
STATE = await res.json();
|
||||||
render();
|
render();
|
||||||
}
|
}
|
||||||
|
|
||||||
async function post(url, body) {
|
async function post(url, body) {
|
||||||
const res = await fetch(url, {
|
let res;
|
||||||
|
try {
|
||||||
|
res = await fetch(url, {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers: {"Content-Type": "application/json"},
|
headers: {"Content-Type": "application/json"},
|
||||||
body: JSON.stringify(body),
|
body: JSON.stringify(body),
|
||||||
});
|
});
|
||||||
|
} catch (err) {
|
||||||
|
alert("That didn't save (no response from the server): " + err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const detail = (await res.json()).detail || res.statusText;
|
const detail = await res.json().then(d => d.detail).catch(() => res.statusText);
|
||||||
alert("That didn't save: " + detail);
|
alert("That didn't save: " + detail);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -298,7 +323,7 @@ function matchCard(row, idx) {
|
|||||||
· rank ${esc(c.rank ?? "—")} · owned ${esc(c.owned ?? "—")}</span></span>
|
· rank ${esc(c.rank ?? "—")} · owned ${esc(c.owned ?? "—")}</span></span>
|
||||||
</li>`).join("");
|
</li>`).join("");
|
||||||
return `
|
return `
|
||||||
<section class="card actionable" data-kind="match" data-idx="${idx}"
|
<section class="card actionable" data-kind="match" data-rowix="${row.row_ix}"
|
||||||
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
|
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
|
||||||
${shots(row.photos)}
|
${shots(row.photos)}
|
||||||
<div class="body">
|
<div class="body">
|
||||||
@@ -324,7 +349,7 @@ function versionCard(row, idx) {
|
|||||||
· ${esc((v.languages || []).join(", "))} · score ${esc(v.score ?? "—")}</span></span>
|
· ${esc((v.languages || []).join(", "))} · score ${esc(v.score ?? "—")}</span></span>
|
||||||
</li>`).join("");
|
</li>`).join("");
|
||||||
return `
|
return `
|
||||||
<section class="card actionable" data-kind="version" data-idx="${idx}"
|
<section class="card actionable" data-kind="version" data-rowix="${row.row_ix}"
|
||||||
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
|
data-title="${esc(row.title_raw)}" data-photos="${esc(row.source_photos)}">
|
||||||
<div class="body">
|
<div class="body">
|
||||||
<p class="title">${esc(row.bgg_name || row.title_raw)}</p>
|
<p class="title">${esc(row.bgg_name || row.title_raw)}</p>
|
||||||
@@ -361,6 +386,8 @@ function ticket(s) {
|
|||||||
function render() {
|
function render() {
|
||||||
const m = document.getElementById("main");
|
const m = document.getElementById("main");
|
||||||
const s = STATE;
|
const s = STATE;
|
||||||
|
showBanner((s.warnings || []).length
|
||||||
|
? `<div class="banner warn">${s.warnings.map(esc).join("<br>")}</div>` : "");
|
||||||
document.getElementById("tally").innerHTML =
|
document.getElementById("tally").innerHTML =
|
||||||
`<span><b>${s.pending.length}</b> matches</span>
|
`<span><b>${s.pending.length}</b> matches</span>
|
||||||
<span><b>${s.versions.length}</b> editions</span>
|
<span><b>${s.versions.length}</b> editions</span>
|
||||||
@@ -480,13 +507,14 @@ function highlight() {
|
|||||||
if (el) el.scrollIntoView({block: "nearest", behavior: "auto"});
|
if (el) el.scrollIntoView({block: "nearest", behavior: "auto"});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const rowIx = card => card.dataset.rowix === undefined ? null : Number(card.dataset.rowix);
|
||||||
const decide = (card, action, bgg_id = null) => post("/api/decision", {
|
const decide = (card, action, bgg_id = null) => post("/api/decision", {
|
||||||
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
||||||
action, bgg_id,
|
row_ix: rowIx(card), action, bgg_id,
|
||||||
});
|
});
|
||||||
const version = (card, action, version_id = null) => post("/api/version", {
|
const version = (card, action, version_id = null) => post("/api/version", {
|
||||||
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
||||||
action, version_id,
|
row_ix: rowIx(card), action, version_id,
|
||||||
});
|
});
|
||||||
const dismiss = t => post("/api/dismiss", {
|
const dismiss = t => post("/api/dismiss", {
|
||||||
photo: t.dataset.photo, location: t.dataset.location,
|
photo: t.dataset.photo, location: t.dataset.location,
|
||||||
@@ -519,23 +547,31 @@ document.addEventListener("keydown", e => {
|
|||||||
else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); }
|
else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); }
|
||||||
});
|
});
|
||||||
|
|
||||||
refresh();
|
refresh().catch(err => errorBanner(err.message || err));
|
||||||
|
|
||||||
// Live-follow the data files: extract/resolve runs in another terminal show
|
// Live-follow the data files: extract/resolve runs in another terminal show
|
||||||
// up on the next poll. Only re-render on an actual change (keeps the
|
// up on the next poll. Only re-render on an actual change (keeps the
|
||||||
// keyboard cursor stable) and never mid-typing in a manual-ID input.
|
// keyboard cursor stable) and never mid-typing in a manual-ID input.
|
||||||
|
let pollMisses = 0;
|
||||||
setInterval(async () => {
|
setInterval(async () => {
|
||||||
const el = document.activeElement;
|
const el = document.activeElement;
|
||||||
if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA")) return;
|
if (el && (el.tagName === "INPUT" || el.tagName === "TEXTAREA")) return;
|
||||||
|
let fresh = null;
|
||||||
try {
|
try {
|
||||||
const fresh = await (await fetch("/api/state")).json();
|
const res = await fetch("/api/state");
|
||||||
|
if (!res.ok) throw new Error(`${res.status}`);
|
||||||
|
fresh = await res.json();
|
||||||
|
} catch (err) {
|
||||||
|
// transient during --dev restarts; persistent means the server is gone
|
||||||
|
if (++pollMisses >= 3) errorBanner(`lost contact (${err.message || err})`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (pollMisses >= 3) showBanner(""); // recovered: clear the lost-contact banner
|
||||||
|
pollMisses = 0;
|
||||||
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
|
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
|
||||||
STATE = fresh;
|
STATE = fresh;
|
||||||
render();
|
render();
|
||||||
}
|
}
|
||||||
} catch {
|
|
||||||
/* server restarting (--dev) — retry on the next tick */
|
|
||||||
}
|
|
||||||
}, 3000);
|
}, 3000);
|
||||||
</script>
|
</script>
|
||||||
</body>
|
</body>
|
||||||
|
|||||||
+82
-25
@@ -27,6 +27,7 @@ import os
|
|||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
|
from collections import Counter
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
@@ -35,7 +36,7 @@ from typing import Protocol
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGAuthError, BGGClient
|
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.models import CollectionItem
|
from bggpipe.models import CollectionItem
|
||||||
|
|
||||||
@@ -51,7 +52,7 @@ UPLOAD_LOG_COLUMNS = [
|
|||||||
"timestamp",
|
"timestamp",
|
||||||
"error",
|
"error",
|
||||||
]
|
]
|
||||||
DONE_STATUSES = {"added", "updated", "already_present"}
|
DONE_STATUSES = {"added", "added_no_version", "updated", "already_present"}
|
||||||
MAX_VERSION_PAGES = 40
|
MAX_VERSION_PAGES = 40
|
||||||
|
|
||||||
|
|
||||||
@@ -73,7 +74,7 @@ class UploadJob:
|
|||||||
return ("add", self.bgg_id, self.version_id)
|
return ("add", self.bgg_id, self.version_id)
|
||||||
|
|
||||||
|
|
||||||
def _row_key(row: dict) -> tuple[str, str, str]:
|
def _job_key(row: dict) -> tuple[str, str, str]:
|
||||||
if row["action"] == "update":
|
if row["action"] == "update":
|
||||||
return ("update", row["collid"], "")
|
return ("update", row["collid"], "")
|
||||||
return ("add", row["bgg_id"], row["version_id"])
|
return ("add", row["bgg_id"], row["version_id"])
|
||||||
@@ -109,7 +110,7 @@ def build_queue(
|
|||||||
done. Returns (jobs, skipped_done, skipped_failed)."""
|
done. Returns (jobs, skipped_done, skipped_failed)."""
|
||||||
latest: dict[tuple[str, str, str], str] = {}
|
latest: dict[tuple[str, str, str], str] = {}
|
||||||
for row in log_rows:
|
for row in log_rows:
|
||||||
latest[_row_key(row)] = row["status"]
|
latest[_job_key(row)] = row["status"]
|
||||||
|
|
||||||
candidates = [
|
candidates = [
|
||||||
UploadJob(
|
UploadJob(
|
||||||
@@ -134,15 +135,25 @@ def build_queue(
|
|||||||
|
|
||||||
jobs: list[UploadJob] = []
|
jobs: list[UploadJob] = []
|
||||||
skipped_done = skipped_failed = 0
|
skipped_done = skipped_failed = 0
|
||||||
|
deferred: list[UploadJob] = []
|
||||||
|
update_game_seen: set[str] = set()
|
||||||
for job in candidates:
|
for job in candidates:
|
||||||
status = latest.get(job.key)
|
status = latest.get(job.key)
|
||||||
if status in DONE_STATUSES:
|
if status in DONE_STATUSES:
|
||||||
skipped_done += 1
|
skipped_done += 1
|
||||||
elif status == "failed" and not retry_failed:
|
elif status == "failed" and not retry_failed:
|
||||||
skipped_failed += 1
|
skipped_failed += 1
|
||||||
|
elif job.action == "update" and job.bgg_id in update_game_seen:
|
||||||
|
# The row-edit flow finds rows by game name, not collid — a
|
||||||
|
# second same-game update this run could reopen the copy the
|
||||||
|
# first one just versioned and overwrite it. One per run; the
|
||||||
|
# next run (after --verify) picks up the rest.
|
||||||
|
deferred.append(job)
|
||||||
else:
|
else:
|
||||||
|
if job.action == "update":
|
||||||
|
update_game_seen.add(job.bgg_id)
|
||||||
jobs.append(job)
|
jobs.append(job)
|
||||||
return jobs, skipped_done, skipped_failed
|
return jobs, skipped_done, skipped_failed, deferred
|
||||||
|
|
||||||
|
|
||||||
def _scrub(text: str) -> str:
|
def _scrub(text: str) -> str:
|
||||||
@@ -155,6 +166,12 @@ def _scrub(text: str) -> str:
|
|||||||
return text
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
class LoginError(RuntimeError):
|
||||||
|
"""Authentication is broken (bad credentials, Cloudflare block, changed
|
||||||
|
login page). Systemic by definition: retrying per-game would hammer the
|
||||||
|
login endpoint and poison upload_log.csv with misleading failures."""
|
||||||
|
|
||||||
|
|
||||||
class Uploader(Protocol):
|
class Uploader(Protocol):
|
||||||
def add_game(self, job: UploadJob) -> tuple[str, str]: ...
|
def add_game(self, job: UploadJob) -> tuple[str, str]: ...
|
||||||
|
|
||||||
@@ -215,11 +232,16 @@ class PlaywrightUploader:
|
|||||||
return
|
return
|
||||||
page = self._page
|
page = self._page
|
||||||
self._goto(f"{BGG}/")
|
self._goto(f"{BGG}/")
|
||||||
|
if "just a moment" in (page.title() or "").casefold():
|
||||||
|
raise LoginError(
|
||||||
|
"Cloudflare is challenging this browser ('Just a moment...') "
|
||||||
|
"— run headed (drop --headless) and click the widget once"
|
||||||
|
)
|
||||||
if self._signed_out():
|
if self._signed_out():
|
||||||
user = os.environ.get("BGG_USERNAME", "")
|
user = os.environ.get("BGG_USERNAME", "")
|
||||||
password = os.environ.get("BGG_PASSWORD", "")
|
password = os.environ.get("BGG_PASSWORD", "")
|
||||||
if not (user and password):
|
if not (user and password):
|
||||||
raise RuntimeError(
|
raise LoginError(
|
||||||
"BGG_USERNAME and BGG_PASSWORD env vars are required to log in"
|
"BGG_USERNAME and BGG_PASSWORD env vars are required to log in"
|
||||||
)
|
)
|
||||||
self._goto(f"{BGG}/login")
|
self._goto(f"{BGG}/login")
|
||||||
@@ -229,7 +251,13 @@ class PlaywrightUploader:
|
|||||||
page.locator("#inputUsername").fill(user)
|
page.locator("#inputUsername").fill(user)
|
||||||
page.locator("#inputPassword").fill(password)
|
page.locator("#inputPassword").fill(password)
|
||||||
page.get_by_role("button", name="Sign In").click()
|
page.get_by_role("button", name="Sign In").click()
|
||||||
|
try:
|
||||||
page.wait_for_url(lambda url: "/login" not in url, timeout=120_000)
|
page.wait_for_url(lambda url: "/login" not in url, timeout=120_000)
|
||||||
|
except self._timeout_error as err:
|
||||||
|
raise LoginError(
|
||||||
|
"login did not complete (wrong credentials, or the login "
|
||||||
|
"page changed — see docs/bgg-upload-flow.md)"
|
||||||
|
) from err
|
||||||
self._context.storage_state(path=str(self._storage_state))
|
self._context.storage_state(path=str(self._storage_state))
|
||||||
self._authed = True
|
self._authed = True
|
||||||
|
|
||||||
@@ -266,6 +294,7 @@ class PlaywrightUploader:
|
|||||||
if nxt.count() == 0 or nxt.is_disabled():
|
if nxt.count() == 0 or nxt.is_disabled():
|
||||||
break
|
break
|
||||||
nxt.click()
|
nxt.click()
|
||||||
|
self._page.wait_for_timeout(2_000) # etiquette: paginating hits BGG too
|
||||||
# Two-level dismissal: the sub-view has its own Cancel distinct from
|
# Two-level dismissal: the sub-view has its own Cancel distinct from
|
||||||
# the main dialog's.
|
# the main dialog's.
|
||||||
dialog.get_by_role("button", name="Cancel").first.click()
|
dialog.get_by_role("button", name="Cancel").first.click()
|
||||||
@@ -283,13 +312,14 @@ class PlaywrightUploader:
|
|||||||
"heading", name=re.compile(re.escape(job.name), re.I)
|
"heading", name=re.compile(re.escape(job.name), re.I)
|
||||||
).wait_for(timeout=15_000)
|
).wait_for(timeout=15_000)
|
||||||
dialog.get_by_label("Own").check()
|
dialog.get_by_label("Own").check()
|
||||||
note = ""
|
status, note = "added", ""
|
||||||
if job.version_name and not self._select_version(dialog, job.version_name):
|
if job.version_name and not self._select_version(dialog, job.version_name):
|
||||||
|
status = "added_no_version"
|
||||||
note = f"version {job.version_name!r} not in picker; added without version"
|
note = f"version {job.version_name!r} not in picker; added without version"
|
||||||
dialog.get_by_role("button", name="Save").click()
|
dialog.get_by_role("button", name="Save").click()
|
||||||
# The dialog is hidden after save, not removed from the DOM.
|
# The dialog is hidden after save, not removed from the DOM.
|
||||||
dialog.wait_for(state="hidden", timeout=15_000)
|
dialog.wait_for(state="hidden", timeout=15_000)
|
||||||
return "added", note
|
return status, note
|
||||||
|
|
||||||
def update_entry(self, job: UploadJob) -> tuple[str, str]:
|
def update_entry(self, job: UploadJob) -> tuple[str, str]:
|
||||||
"""Set the version on an EXISTING entry — strictly additive.
|
"""Set the version on an EXISTING entry — strictly additive.
|
||||||
@@ -332,6 +362,7 @@ def _process(
|
|||||||
now: Callable[[], str],
|
now: Callable[[], str],
|
||||||
) -> list[dict]:
|
) -> list[dict]:
|
||||||
results = []
|
results = []
|
||||||
|
consecutive: tuple[str, int] = ("", 0)
|
||||||
for i, job in enumerate(jobs):
|
for i, job in enumerate(jobs):
|
||||||
if i:
|
if i:
|
||||||
sleep(rng.uniform(2.0, 4.0)) # polite pacing between games (spec)
|
sleep(rng.uniform(2.0, 4.0)) # polite pacing between games (spec)
|
||||||
@@ -340,6 +371,12 @@ def _process(
|
|||||||
status, note = uploader.add_game(job)
|
status, note = uploader.add_game(job)
|
||||||
else:
|
else:
|
||||||
status, note = uploader.update_entry(job)
|
status, note = uploader.update_entry(job)
|
||||||
|
except LoginError as exc:
|
||||||
|
# Systemic: every remaining job would fail identically. Abort
|
||||||
|
# WITHOUT logging failures, so the next run just retries.
|
||||||
|
typer.echo(f" aborting — login is broken: {_scrub(str(exc))}")
|
||||||
|
typer.echo(f" {len(jobs) - i} job(s) left untouched for the next run.")
|
||||||
|
break
|
||||||
except Exception as exc: # per-game isolation: log it, keep going
|
except Exception as exc: # per-game isolation: log it, keep going
|
||||||
status, note = "failed", _scrub(f"{type(exc).__name__}: {exc}")
|
status, note = "failed", _scrub(f"{type(exc).__name__}: {exc}")
|
||||||
row = {
|
row = {
|
||||||
@@ -350,12 +387,25 @@ def _process(
|
|||||||
"version_id": job.version_id,
|
"version_id": job.version_id,
|
||||||
"status": status,
|
"status": status,
|
||||||
"timestamp": now(),
|
"timestamp": now(),
|
||||||
"error": note if status == "failed" else note,
|
# the column carries degradation notes on successes too
|
||||||
|
# (e.g. added_no_version), not just failure text
|
||||||
|
"error": note,
|
||||||
}
|
}
|
||||||
append_log_row(log_path, row)
|
append_log_row(log_path, row)
|
||||||
results.append(row)
|
results.append(row)
|
||||||
suffix = f" — {note}" if note else ""
|
suffix = f" — {note}" if note else ""
|
||||||
typer.echo(f" {job.name}: {status}{suffix}")
|
typer.echo(f" {job.name}: {status}{suffix}")
|
||||||
|
if status == "failed":
|
||||||
|
consecutive = (note, consecutive[1] + 1 if note == consecutive[0] else 1)
|
||||||
|
if consecutive[1] >= 3:
|
||||||
|
typer.echo(
|
||||||
|
" aborting — 3 identical consecutive failures look "
|
||||||
|
"systemic, not per-game; remaining jobs left for the "
|
||||||
|
"next run"
|
||||||
|
)
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
consecutive = ("", 0)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@@ -370,14 +420,18 @@ def verify_uploads(log_rows: list[dict], collection: list[CollectionItem]) -> li
|
|||||||
problems = []
|
problems = []
|
||||||
latest: dict[tuple[str, str, str], dict] = {}
|
latest: dict[tuple[str, str, str], dict] = {}
|
||||||
for row in log_rows:
|
for row in log_rows:
|
||||||
latest[_row_key(row)] = row
|
latest[_job_key(row)] = row
|
||||||
for row in latest.values():
|
for row in latest.values():
|
||||||
if row["status"] == "added":
|
if row["status"] in ("added", "added_no_version"):
|
||||||
copies = by_object.get(int(row["bgg_id"]), [])
|
copies = by_object.get(int(row["bgg_id"]), [])
|
||||||
if not copies:
|
if not copies:
|
||||||
problems.append(f"{row['name']}: logged added but not in collection")
|
problems.append(f"{row['name']}: logged added but not in collection")
|
||||||
elif row["version_id"] and not any(
|
elif (
|
||||||
|
row["status"] == "added" # no_version: absence is expected
|
||||||
|
and row["version_id"]
|
||||||
|
and not any(
|
||||||
str(c.version_id or "") == row["version_id"] for c in copies
|
str(c.version_id or "") == row["version_id"] for c in copies
|
||||||
|
)
|
||||||
):
|
):
|
||||||
problems.append(
|
problems.append(
|
||||||
f"{row['name']}: in collection but no copy has "
|
f"{row['name']}: in collection but no copy has "
|
||||||
@@ -419,11 +473,7 @@ def run_upload(
|
|||||||
# cache marker travels with the stub XML (gitignored, so a fresh clone
|
# cache marker travels with the stub XML (gitignored, so a fresh clone
|
||||||
# loses it), while data/STUB_DATA.marker is COMMITTED alongside the
|
# loses it), while data/STUB_DATA.marker is COMMITTED alongside the
|
||||||
# stub-derived CSVs — so a clone can never upload placeholder ids.
|
# stub-derived CSVs — so a clone can never upload placeholder ids.
|
||||||
markers = [
|
marker = next((m for m in cfg.stub_marker_paths if m.exists()), None)
|
||||||
cfg.cache_dir / "STUB_FIXTURES.marker",
|
|
||||||
cfg.data_dir / "STUB_DATA.marker",
|
|
||||||
]
|
|
||||||
marker = next((m for m in markers if m.exists()), None)
|
|
||||||
if marker is not None:
|
if marker is not None:
|
||||||
if not dry_run:
|
if not dry_run:
|
||||||
typer.echo(
|
typer.echo(
|
||||||
@@ -439,12 +489,15 @@ def run_upload(
|
|||||||
"placeholders, not real BGG data.\n"
|
"placeholders, not real BGG data.\n"
|
||||||
)
|
)
|
||||||
|
|
||||||
log_path = cfg.data_dir / "upload_log.csv"
|
log_path = cfg.upload_log_path
|
||||||
to_add = _read_csv(cfg.data_dir / "to_add.csv")
|
if not cfg.to_add_path.exists():
|
||||||
to_update = _read_csv(cfg.data_dir / "to_update.csv")
|
typer.echo(f"{cfg.to_add_path} not found — run `bggpipe diff` first.")
|
||||||
|
raise typer.Exit(code=1)
|
||||||
|
to_add = _read_csv(cfg.to_add_path)
|
||||||
|
to_update = _read_csv(cfg.to_update_path)
|
||||||
log_rows = _read_csv(log_path)
|
log_rows = _read_csv(log_path)
|
||||||
|
|
||||||
jobs, skipped_done, skipped_failed = build_queue(
|
jobs, skipped_done, skipped_failed, deferred = build_queue(
|
||||||
to_add, to_update, log_rows, retry_failed=retry_failed
|
to_add, to_update, log_rows, retry_failed=retry_failed
|
||||||
)
|
)
|
||||||
if limit is not None:
|
if limit is not None:
|
||||||
@@ -456,6 +509,12 @@ def run_upload(
|
|||||||
f" (skipping {skipped_done} already done, {skipped_failed} "
|
f" (skipping {skipped_done} already done, {skipped_failed} "
|
||||||
"previously failed — use --retry-failed)"
|
"previously failed — use --retry-failed)"
|
||||||
)
|
)
|
||||||
|
if deferred:
|
||||||
|
typer.echo(
|
||||||
|
f"{len(deferred)} version update(s) deferred: one update per game "
|
||||||
|
"per run (the row-edit flow can't target a collid) — re-run "
|
||||||
|
"upload after --verify confirms this batch."
|
||||||
|
)
|
||||||
|
|
||||||
results: list[dict] = []
|
results: list[dict] = []
|
||||||
if dry_run:
|
if dry_run:
|
||||||
@@ -484,9 +543,7 @@ def run_upload(
|
|||||||
results = _process(real, jobs, log_path, sleep=sleep, rng=rng, now=now)
|
results = _process(real, jobs, log_path, sleep=sleep, rng=rng, now=now)
|
||||||
else:
|
else:
|
||||||
results = _process(uploader, jobs, log_path, sleep=sleep, rng=rng, now=now)
|
results = _process(uploader, jobs, log_path, sleep=sleep, rng=rng, now=now)
|
||||||
counts: dict[str, int] = {}
|
counts = Counter(row["status"] for row in results)
|
||||||
for row in results:
|
|
||||||
counts[row["status"]] = counts.get(row["status"], 0) + 1
|
|
||||||
summary = " · ".join(f"{n} {status}" for status, n in sorted(counts.items()))
|
summary = " · ".join(f"{n} {status}" for status, n in sorted(counts.items()))
|
||||||
typer.echo(f"\n{summary or 'nothing to do'}")
|
typer.echo(f"\n{summary or 'nothing to do'}")
|
||||||
else:
|
else:
|
||||||
@@ -501,7 +558,7 @@ def _run_verify(cfg: Config, client: BGGClient | None, log_path: Path) -> None:
|
|||||||
if not cfg.bgg_username:
|
if not cfg.bgg_username:
|
||||||
typer.echo("--verify needs BGG_USERNAME in the environment.")
|
typer.echo("--verify needs BGG_USERNAME in the environment.")
|
||||||
return
|
return
|
||||||
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
client = client or client_for(cfg)
|
||||||
try:
|
try:
|
||||||
collection = client.collection_full(cfg.bgg_username, refresh=True)
|
collection = client.collection_full(cfg.bgg_username, refresh=True)
|
||||||
except BGGAuthError as exc:
|
except BGGAuthError as exc:
|
||||||
|
|||||||
+45
-15
@@ -15,6 +15,8 @@ from __future__ import annotations
|
|||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
|
from collections import Counter
|
||||||
from importlib import resources
|
from importlib import resources
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -26,11 +28,10 @@ from pydantic import BaseModel
|
|||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGClient
|
from bggpipe.bgg_client import BGGClient
|
||||||
|
from bggpipe.config import DEFAULT_REVIEW_PORT as DEFAULT_PORT # single home
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.review import ReviewSession
|
from bggpipe.review import ReviewSession
|
||||||
|
|
||||||
DEFAULT_PORT = 8377
|
|
||||||
|
|
||||||
|
|
||||||
def load_thumbnails(cache_dir: Path) -> dict[int, str]:
|
def load_thumbnails(cache_dir: Path) -> dict[int, str]:
|
||||||
"""bgg_id -> thumbnail URL, from cached /thing XML only (no live calls).
|
"""bgg_id -> thumbnail URL, from cached /thing XML only (no live calls).
|
||||||
@@ -41,7 +42,7 @@ def load_thumbnails(cache_dir: Path) -> dict[int, str]:
|
|||||||
for path in cache_dir.glob("thing_*.xml"):
|
for path in cache_dir.glob("thing_*.xml"):
|
||||||
try:
|
try:
|
||||||
root = _safe_fromstring(path.read_text())
|
root = _safe_fromstring(path.read_text())
|
||||||
except Exception: # noqa: BLE001 — a corrupt cache file must not kill the UI
|
except Exception: # a corrupt cache file must not kill the UI
|
||||||
continue
|
continue
|
||||||
for item in root.findall("item"):
|
for item in root.findall("item"):
|
||||||
thumb = (item.findtext("thumbnail") or "").strip()
|
thumb = (item.findtext("thumbnail") or "").strip()
|
||||||
@@ -80,6 +81,7 @@ class DismissStore:
|
|||||||
class DecisionBody(BaseModel):
|
class DecisionBody(BaseModel):
|
||||||
title_raw: str
|
title_raw: str
|
||||||
source_photos: str
|
source_photos: str
|
||||||
|
row_ix: int | None = None
|
||||||
action: str # "pick" | "manual" | "reject"
|
action: str # "pick" | "manual" | "reject"
|
||||||
bgg_id: int | None = None
|
bgg_id: int | None = None
|
||||||
|
|
||||||
@@ -87,6 +89,7 @@ class DecisionBody(BaseModel):
|
|||||||
class VersionBody(BaseModel):
|
class VersionBody(BaseModel):
|
||||||
title_raw: str
|
title_raw: str
|
||||||
source_photos: str
|
source_photos: str
|
||||||
|
row_ix: int | None = None
|
||||||
action: str # "pick" | "unknown"
|
action: str # "pick" | "unknown"
|
||||||
version_id: int | None = None
|
version_id: int | None = None
|
||||||
|
|
||||||
@@ -94,6 +97,7 @@ class VersionBody(BaseModel):
|
|||||||
class VetoBody(BaseModel):
|
class VetoBody(BaseModel):
|
||||||
title_raw: str
|
title_raw: str
|
||||||
source_photos: str
|
source_photos: str
|
||||||
|
row_ix: int | None = None
|
||||||
|
|
||||||
|
|
||||||
class DismissBody(BaseModel):
|
class DismissBody(BaseModel):
|
||||||
@@ -112,7 +116,11 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
client=client,
|
client=client,
|
||||||
)
|
)
|
||||||
thumbnails = load_thumbnails(cfg.cache_dir)
|
thumbnails = load_thumbnails(cfg.cache_dir)
|
||||||
dismissed = DismissStore(cfg.data_dir / "unidentified_dismissed.json")
|
dismissed = DismissStore(cfg.dismissed_path)
|
||||||
|
# FastAPI runs sync endpoints in a threadpool: without this, a polled
|
||||||
|
# GET /api/state can freshen()-swap session.rows out from under a
|
||||||
|
# concurrent decision POST, silently dropping the decision.
|
||||||
|
lock = threading.Lock()
|
||||||
|
|
||||||
def freshen() -> None:
|
def freshen() -> None:
|
||||||
"""Serve every request from the current file state: an extract or
|
"""Serve every request from the current file state: an extract or
|
||||||
@@ -121,8 +129,16 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
if session.reload_if_changed():
|
if session.reload_if_changed():
|
||||||
thumbnails = load_thumbnails(cfg.cache_dir)
|
thumbnails = load_thumbnails(cfg.cache_dir)
|
||||||
|
|
||||||
def find_row(title_raw: str, source_photos: str) -> dict:
|
def find_row(
|
||||||
|
title_raw: str, source_photos: str, row_ix: int | None = None
|
||||||
|
) -> dict:
|
||||||
freshen()
|
freshen()
|
||||||
|
# ordinal first: (title_raw, source_photos) is not unique when one
|
||||||
|
# photo holds two editions of the same game
|
||||||
|
if row_ix is not None and 0 <= row_ix < len(session.rows):
|
||||||
|
row = session.rows[row_ix]
|
||||||
|
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
|
||||||
|
return row
|
||||||
for row in session.rows:
|
for row in session.rows:
|
||||||
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
|
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
|
||||||
return row
|
return row
|
||||||
@@ -134,12 +150,13 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
return {p.name for p in cfg.photos_dir.iterdir() if p.is_file()}
|
return {p.name for p in cfg.photos_dir.iterdir() if p.is_file()}
|
||||||
|
|
||||||
def row_payload(row: dict) -> dict:
|
def row_payload(row: dict) -> dict:
|
||||||
entry = session.cues_for(row["title_raw"])
|
entry = session.cues_for(row["title_raw"], row["source_photos"])
|
||||||
available = photo_names()
|
available = photo_names()
|
||||||
candidates = json.loads(row["candidates_json"] or "[]")
|
candidates = json.loads(row["candidates_json"] or "[]")
|
||||||
for c in candidates:
|
for c in candidates:
|
||||||
c["thumbnail"] = thumbnails.get(c.get("bgg_id"))
|
c["thumbnail"] = thumbnails.get(c.get("bgg_id"))
|
||||||
return {
|
return {
|
||||||
|
"row_ix": session.rows.index(row),
|
||||||
"title_raw": row["title_raw"],
|
"title_raw": row["title_raw"],
|
||||||
"source_photos": row["source_photos"],
|
"source_photos": row["source_photos"],
|
||||||
"match_status": row["match_status"],
|
"match_status": row["match_status"],
|
||||||
@@ -156,6 +173,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
def version_payload(row: dict) -> dict:
|
def version_payload(row: dict) -> dict:
|
||||||
return {
|
return {
|
||||||
|
"row_ix": session.rows.index(row),
|
||||||
"title_raw": row["title_raw"],
|
"title_raw": row["title_raw"],
|
||||||
"source_photos": row["source_photos"],
|
"source_photos": row["source_photos"],
|
||||||
"bgg_name": row["bgg_name"],
|
"bgg_name": row["bgg_name"],
|
||||||
@@ -164,9 +182,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
def state() -> dict:
|
def state() -> dict:
|
||||||
freshen()
|
freshen()
|
||||||
counts: dict[str, int] = {}
|
counts = Counter(row["match_status"] for row in session.rows)
|
||||||
for row in session.rows:
|
|
||||||
counts[row["match_status"]] = counts.get(row["match_status"], 0) + 1
|
|
||||||
resolved_titles = {r["title_raw"] for r in session.rows}
|
resolved_titles = {r["title_raw"] for r in session.rows}
|
||||||
rows_by_title: dict[str, dict] = {}
|
rows_by_title: dict[str, dict] = {}
|
||||||
for r in session.rows:
|
for r in session.rows:
|
||||||
@@ -216,6 +232,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
for r in session.merged_rows()
|
for r in session.merged_rows()
|
||||||
]
|
]
|
||||||
return {
|
return {
|
||||||
|
"warnings": session.warnings[-10:],
|
||||||
"pending": [row_payload(r) for r in session.pending_rows()],
|
"pending": [row_payload(r) for r in session.pending_rows()],
|
||||||
"versions": [version_payload(r) for r in session.version_rows()],
|
"versions": [version_payload(r) for r in session.version_rows()],
|
||||||
"merges": merges,
|
"merges": merges,
|
||||||
@@ -240,11 +257,16 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
@app.get("/api/state")
|
@app.get("/api/state")
|
||||||
def api_state() -> dict:
|
def api_state() -> dict:
|
||||||
|
with lock:
|
||||||
return state()
|
return state()
|
||||||
|
|
||||||
@app.post("/api/decision")
|
@app.post("/api/decision")
|
||||||
def api_decision(body: DecisionBody) -> dict:
|
def api_decision(body: DecisionBody) -> dict:
|
||||||
row = find_row(body.title_raw, body.source_photos)
|
with lock:
|
||||||
|
return _decide(body)
|
||||||
|
|
||||||
|
def _decide(body: DecisionBody) -> dict:
|
||||||
|
row = find_row(body.title_raw, body.source_photos, body.row_ix)
|
||||||
if body.action == "pick":
|
if body.action == "pick":
|
||||||
candidates = json.loads(row["candidates_json"] or "[]")
|
candidates = json.loads(row["candidates_json"] or "[]")
|
||||||
chosen = next(
|
chosen = next(
|
||||||
@@ -265,7 +287,11 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
@app.post("/api/version")
|
@app.post("/api/version")
|
||||||
def api_version(body: VersionBody) -> dict:
|
def api_version(body: VersionBody) -> dict:
|
||||||
row = find_row(body.title_raw, body.source_photos)
|
with lock:
|
||||||
|
return _version(body)
|
||||||
|
|
||||||
|
def _version(body: VersionBody) -> dict:
|
||||||
|
row = find_row(body.title_raw, body.source_photos, body.row_ix)
|
||||||
if body.action == "unknown":
|
if body.action == "unknown":
|
||||||
session.decide_version(row, None)
|
session.decide_version(row, None)
|
||||||
elif body.action == "pick":
|
elif body.action == "pick":
|
||||||
@@ -279,7 +305,8 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
@app.post("/api/veto-merge")
|
@app.post("/api/veto-merge")
|
||||||
def api_veto_merge(body: VetoBody) -> dict:
|
def api_veto_merge(body: VetoBody) -> dict:
|
||||||
row = find_row(body.title_raw, body.source_photos)
|
with lock:
|
||||||
|
row = find_row(body.title_raw, body.source_photos, body.row_ix)
|
||||||
if row["match_status"] != "merged":
|
if row["match_status"] != "merged":
|
||||||
raise HTTPException(400, "row is not merged")
|
raise HTTPException(400, "row is not merged")
|
||||||
session.veto_merge(row)
|
session.veto_merge(row)
|
||||||
@@ -287,11 +314,14 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
@app.post("/api/dismiss")
|
@app.post("/api/dismiss")
|
||||||
def api_dismiss(body: DismissBody) -> dict:
|
def api_dismiss(body: DismissBody) -> dict:
|
||||||
dismissed.add(_sighting_key(body.photo, body.model_dump(exclude={"photo"})))
|
with lock:
|
||||||
|
dismissed.add(
|
||||||
|
_sighting_key(body.photo, body.model_dump(exclude={"photo"}))
|
||||||
|
)
|
||||||
return state()
|
return state()
|
||||||
|
|
||||||
@app.get("/photos/{name}")
|
@app.get("/photos/{name}")
|
||||||
def photo(name: str):
|
def photo(name: str) -> FileResponse:
|
||||||
if name not in photo_names(): # also blocks any path traversal
|
if name not in photo_names(): # also blocks any path traversal
|
||||||
raise HTTPException(404, "no such photo")
|
raise HTTPException(404, "no such photo")
|
||||||
return FileResponse(cfg.photos_dir / name)
|
return FileResponse(cfg.photos_dir / name)
|
||||||
@@ -303,7 +333,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
}
|
}
|
||||||
|
|
||||||
@app.get("/static/{name}")
|
@app.get("/static/{name}")
|
||||||
def static_asset(name: str):
|
def static_asset(name: str) -> Response:
|
||||||
media_type = _STATIC.get(name) # allowlist: no traversal possible
|
media_type = _STATIC.get(name) # allowlist: no traversal possible
|
||||||
if media_type is None:
|
if media_type is None:
|
||||||
raise HTTPException(404, "no such asset")
|
raise HTTPException(404, "no such asset")
|
||||||
|
|||||||
@@ -131,3 +131,54 @@ def test_401_raises_actionable_auth_error(tmp_path, monkeypatch):
|
|||||||
client, _, _ = make_client(tmp_path, [(401, "Unauthorized")])
|
client, _, _ = make_client(tmp_path, [(401, "Unauthorized")])
|
||||||
with pytest.raises(BGGAuthError, match="BGG_API_TOKEN"):
|
with pytest.raises(BGGAuthError, match="BGG_API_TOKEN"):
|
||||||
client.get_xml("search", {"query": "catan"})
|
client.get_xml("search", {"query": "catan"})
|
||||||
|
|
||||||
|
|
||||||
|
# -- audit-fix regressions ----------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_http_200_error_document_raises_and_is_never_cached(tmp_path):
|
||||||
|
# BGG serves some errors as HTTP 200 <errors> XML; caching one would
|
||||||
|
# poison every future run for that query
|
||||||
|
from bggpipe.models import BGGResponseError
|
||||||
|
|
||||||
|
errors_xml = "<errors><error><message>Invalid username</message></error></errors>"
|
||||||
|
client, _, _ = make_client(tmp_path, [(200, errors_xml)])
|
||||||
|
with pytest.raises(BGGResponseError):
|
||||||
|
client.get_xml("collection", {"username": "nobody", "own": "1"})
|
||||||
|
assert list((tmp_path / "cache").glob("*.xml")) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection_full_merges_and_dedupes_by_collid(tmp_path):
|
||||||
|
base_xml = (
|
||||||
|
'<items totalitems="2">'
|
||||||
|
'<item objectid="13" collid="100" subtype="boardgame">'
|
||||||
|
'<name>Catan</name><status own="1"/></item>'
|
||||||
|
'<item objectid="177" collid="101" subtype="boardgame">'
|
||||||
|
'<name>Advanced Civilization</name><status own="1"/></item>'
|
||||||
|
"</items>"
|
||||||
|
)
|
||||||
|
expansion_xml = (
|
||||||
|
'<items totalitems="1">'
|
||||||
|
'<item objectid="177" collid="101" subtype="boardgameexpansion">'
|
||||||
|
'<name>Advanced Civilization</name><status own="1"/></item>'
|
||||||
|
"</items>"
|
||||||
|
)
|
||||||
|
client, _, _ = make_client(tmp_path, [(200, base_xml), (200, expansion_xml)])
|
||||||
|
items = client.collection_full("eric")
|
||||||
|
assert len(items) == 2 # collid 101 appears in both responses: one copy
|
||||||
|
assert {i.coll_id for i in items} == {100, 101}
|
||||||
|
|
||||||
|
|
||||||
|
def test_collection_item_missing_collid_refuses_to_parse(tmp_path):
|
||||||
|
# a truncated response must fail loudly, not coerce ids to 0 and let
|
||||||
|
# the dedupe silently drop owned games
|
||||||
|
from bggpipe.models import BGGResponseError, parse_collection
|
||||||
|
|
||||||
|
bad_xml = (
|
||||||
|
'<items totalitems="1">'
|
||||||
|
'<item objectid="13" subtype="boardgame">'
|
||||||
|
'<name>Catan</name><status own="1"/></item>'
|
||||||
|
"</items>"
|
||||||
|
)
|
||||||
|
with pytest.raises(BGGResponseError):
|
||||||
|
parse_collection(bad_xml)
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
"""Config loading: toml knobs, env-only username, unknown-key warning."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from bggpipe.config import Config, load_config
|
from bggpipe.config import Config, load_config
|
||||||
@@ -29,3 +33,14 @@ def test_username_comes_from_env_only(tmp_path, monkeypatch):
|
|||||||
assert load_config(p).bgg_username == "from_env"
|
assert load_config(p).bgg_username == "from_env"
|
||||||
monkeypatch.delenv("BGG_USERNAME")
|
monkeypatch.delenv("BGG_USERNAME")
|
||||||
assert load_config(p).bgg_username == ""
|
assert load_config(p).bgg_username == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_toml_keys_warn(tmp_path, monkeypatch):
|
||||||
|
# a typo'd knob must not silently fall back to defaults
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
monkeypatch.delenv("BGG_USERNAME", raising=False)
|
||||||
|
p = tmp_path / "config.toml"
|
||||||
|
p.write_text('photo_dir = "oops"\n')
|
||||||
|
with pytest.warns(UserWarning, match="photo_dir"):
|
||||||
|
load_config(p)
|
||||||
|
|||||||
+64
-5
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from bggpipe.config import Config
|
||||||
from bggpipe.diff import compute_diff, load_snapshot_collection
|
from bggpipe.diff import compute_diff, load_snapshot_collection
|
||||||
from bggpipe.models import CollectionItem
|
from bggpipe.models import CollectionItem
|
||||||
|
|
||||||
@@ -114,7 +115,10 @@ def test_owned_with_matching_version_is_just_owned():
|
|||||||
assert not result.to_update and not result.to_add
|
assert not result.to_update and not result.to_add
|
||||||
|
|
||||||
|
|
||||||
def test_owned_with_different_version_reports_disagreement_untouched():
|
def test_confident_version_matching_no_copy_is_a_second_copy_to_add():
|
||||||
|
# Spec: a (bgg_id, version_id) pair is owned only if a collection item
|
||||||
|
# matches BOTH. All copies carry different versions -> this is an
|
||||||
|
# additional physical copy; existing entries are never edited.
|
||||||
result = compute_diff(
|
result = compute_diff(
|
||||||
[
|
[
|
||||||
_match(
|
_match(
|
||||||
@@ -127,9 +131,10 @@ def test_owned_with_different_version_reports_disagreement_untouched():
|
|||||||
],
|
],
|
||||||
[_item(266192, 5, version_id=465063)],
|
[_item(266192, 5, version_id=465063)],
|
||||||
)
|
)
|
||||||
assert result.already_owned == ["Wingspan"]
|
|
||||||
assert not result.to_update # additive only: never edit a set version
|
assert not result.to_update # additive only: never edit a set version
|
||||||
assert "fourth printing" in result.disagreements[0]
|
assert [r["version_id"] for r in result.to_add] == ["521212"]
|
||||||
|
assert "fourth printing" in result.second_copies[0]
|
||||||
|
assert result.already_owned == []
|
||||||
|
|
||||||
|
|
||||||
def test_version_unknown_owned_by_bare_id():
|
def test_version_unknown_owned_by_bare_id():
|
||||||
@@ -169,8 +174,11 @@ def test_pending_rejected_and_unseen_are_reported():
|
|||||||
def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
|
def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
|
||||||
matches = [
|
matches = [
|
||||||
_match("Joking Hazard", "193621"),
|
_match("Joking Hazard", "193621"),
|
||||||
{**_match("Jokin Ha...", "193621", status="merged"),
|
{
|
||||||
"merged_into": "Joking Hazard", "source_photos": "other.jpg"},
|
**_match("Jokin Ha...", "193621", status="merged"),
|
||||||
|
"merged_into": "Joking Hazard",
|
||||||
|
"source_photos": "other.jpg",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
result = compute_diff(matches, []) # empty collection -> to_add
|
result = compute_diff(matches, []) # empty collection -> to_add
|
||||||
assert result.merged == 1
|
assert result.merged == 1
|
||||||
@@ -178,3 +186,54 @@ def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
|
|||||||
(row,) = result.to_add
|
(row,) = result.to_add
|
||||||
assert row["title_raw"] == "Joking Hazard"
|
assert row["title_raw"] == "Joking Hazard"
|
||||||
assert row["source_photos"] == "other.jpg;x.jpg" # combined
|
assert row["source_photos"] == "other.jpg;x.jpg" # combined
|
||||||
|
|
||||||
|
|
||||||
|
def test_versionless_copies_exhaust_then_second_copy_becomes_add():
|
||||||
|
# two confident-version matches, ONE versionless copy: the first consumes
|
||||||
|
# it (to_update), the second is an additional physical copy (to_add)
|
||||||
|
result = compute_diff(
|
||||||
|
[
|
||||||
|
_match("Sorcerer", "39", vstatus="version_auto", vid="111", vname="1st"),
|
||||||
|
_match("Sorcerer", "39", vstatus="version_auto", vid="222", vname="2nd"),
|
||||||
|
],
|
||||||
|
[_item(39, 701)],
|
||||||
|
)
|
||||||
|
assert [u["version_id"] for u in result.to_update] == ["111"]
|
||||||
|
assert [a["version_id"] for a in result.to_add] == ["222"]
|
||||||
|
assert len(result.second_copies) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_diff_outputs_feed_upload_unchanged(tmp_path, monkeypatch):
|
||||||
|
# the cross-stage contract: whatever run_diff writes, run_upload must
|
||||||
|
# read — a column rename on either side has to fail HERE
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from bggpipe.diff import run_diff
|
||||||
|
from bggpipe.resolve import write_matches
|
||||||
|
from bggpipe.upload import run_upload
|
||||||
|
|
||||||
|
monkeypatch.delenv("BGG_API_TOKEN", raising=False)
|
||||||
|
cfg = Config(data_dir=tmp_path)
|
||||||
|
fixtures = Path(__file__).parent / "fixtures"
|
||||||
|
for name in (
|
||||||
|
"collection_snapshot_base.xml",
|
||||||
|
"collection_snapshot_expansions.xml",
|
||||||
|
):
|
||||||
|
shutil.copy(fixtures / name, tmp_path / name)
|
||||||
|
write_matches(
|
||||||
|
cfg.matches_path,
|
||||||
|
[
|
||||||
|
_match("Wingspan", "266192", status="auto"), # not in the snapshots
|
||||||
|
_match("5 MINUTE DUNGEON", "207830", status="auto"), # owned
|
||||||
|
],
|
||||||
|
)
|
||||||
|
result = run_diff(cfg)
|
||||||
|
assert [r["bgg_id"] for r in result.to_add] == ["266192"]
|
||||||
|
|
||||||
|
from test_upload import FakeUploader
|
||||||
|
|
||||||
|
fake = FakeUploader()
|
||||||
|
run_upload(cfg, uploader=fake, sleep=lambda s: None, now=lambda: "t")
|
||||||
|
assert [(j.action, j.bgg_id, j.name) for j in fake.calls] == [
|
||||||
|
("add", "266192", "Wingspan")
|
||||||
|
]
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import base64
|
import base64
|
||||||
import io
|
import io
|
||||||
|
import json
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
import typer
|
import typer
|
||||||
@@ -83,21 +84,21 @@ def test_parse_object_with_titles_and_unidentified():
|
|||||||
"unidentified": [{"location": "top shelf, left of Catan",
|
"unidentified": [{"location": "top shelf, left of Catan",
|
||||||
"partial_text": "WAR", "art_notes": "red spine"}]}
|
"partial_text": "WAR", "art_notes": "red spine"}]}
|
||||||
```"""
|
```"""
|
||||||
titles, unidentified = parse_vision_response(text)
|
titles, unidentified, _ = parse_vision_response(text)
|
||||||
assert titles[0]["title_raw"] == "Catan"
|
assert titles[0]["title_raw"] == "Catan"
|
||||||
assert unidentified[0]["location"] == "top shelf, left of Catan"
|
assert unidentified[0]["location"] == "top shelf, left of Catan"
|
||||||
|
|
||||||
|
|
||||||
def test_parse_legacy_bare_array_still_works():
|
def test_parse_legacy_bare_array_still_works():
|
||||||
text = '```json\n[{"title_raw": "Catan", "confidence": "high"}]\n```'
|
text = '```json\n[{"title_raw": "Catan", "confidence": "high"}]\n```'
|
||||||
titles, unidentified = parse_vision_response(text)
|
titles, unidentified, _ = parse_vision_response(text)
|
||||||
assert titles[0]["title_raw"] == "Catan"
|
assert titles[0]["title_raw"] == "Catan"
|
||||||
assert unidentified == []
|
assert unidentified == []
|
||||||
|
|
||||||
|
|
||||||
def test_parse_tolerates_prose_around_json():
|
def test_parse_tolerates_prose_around_json():
|
||||||
text = 'Here are the games:\n[{"title_raw": "Wingspan"}]\nLet me know!'
|
text = 'Here are the games:\n[{"title_raw": "Wingspan"}]\nLet me know!'
|
||||||
titles, _ = parse_vision_response(text)
|
titles, _, _ = parse_vision_response(text)
|
||||||
assert titles[0]["title_raw"] == "Wingspan"
|
assert titles[0]["title_raw"] == "Wingspan"
|
||||||
|
|
||||||
|
|
||||||
@@ -106,7 +107,7 @@ def test_parse_drops_malformed_entries():
|
|||||||
'{"titles": [{"title_raw": "Catan"}, {"no_title": true}, "just a string"],'
|
'{"titles": [{"title_raw": "Catan"}, {"no_title": true}, "just a string"],'
|
||||||
' "unidentified": [{}, "not a dict", {"location": "somewhere"}]}'
|
' "unidentified": [{}, "not a dict", {"location": "somewhere"}]}'
|
||||||
)
|
)
|
||||||
titles, unidentified = parse_vision_response(text)
|
titles, unidentified, _ = parse_vision_response(text)
|
||||||
assert len(titles) == 1
|
assert len(titles) == 1
|
||||||
assert len(unidentified) == 1 # empty {} and the bare string are dropped
|
assert len(unidentified) == 1 # empty {} and the bare string are dropped
|
||||||
|
|
||||||
@@ -227,7 +228,6 @@ def test_run_extract_empty_photos_dir_exits(tmp_path):
|
|||||||
|
|
||||||
# -- unidentified sightings ---------------------------------------------
|
# -- unidentified sightings ---------------------------------------------
|
||||||
|
|
||||||
import json # noqa: E402
|
|
||||||
|
|
||||||
OBJECT_PAYLOAD = json.dumps(
|
OBJECT_PAYLOAD = json.dumps(
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
"""XML parser tests against hand-built API2 response shapes."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from bggpipe.models import (
|
from bggpipe.models import (
|
||||||
|
|||||||
@@ -1,3 +1,7 @@
|
|||||||
|
"""Title normalization must be symmetric and aggressive (spec)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
from bggpipe.normalize import normalize_title
|
from bggpipe.normalize import normalize_title
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+120
-12
@@ -14,17 +14,23 @@ from pathlib import Path
|
|||||||
import httpx
|
import httpx
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGClient
|
from bggpipe.bgg_client import BGGClient, cache_key
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.models import GameVersion
|
from bggpipe.models import GameVersion
|
||||||
|
from bggpipe.normalize import normalize_title
|
||||||
from bggpipe.resolve import (
|
from bggpipe.resolve import (
|
||||||
Candidate,
|
Candidate,
|
||||||
|
MatchRow,
|
||||||
TitleEntry,
|
TitleEntry,
|
||||||
_dominant,
|
_dominant,
|
||||||
_score_version,
|
_score_version,
|
||||||
|
_truncation_heads,
|
||||||
|
dedupe_matches,
|
||||||
load_titles,
|
load_titles,
|
||||||
|
read_matches,
|
||||||
resolve_entry,
|
resolve_entry,
|
||||||
run_resolve,
|
run_resolve,
|
||||||
|
write_matches,
|
||||||
)
|
)
|
||||||
|
|
||||||
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
|
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
|
||||||
@@ -191,9 +197,6 @@ def test_score_version_no_overlap():
|
|||||||
|
|
||||||
# -- progressive title truncation (long transcribed box titles) ---------
|
# -- progressive title truncation (long transcribed box titles) ---------
|
||||||
|
|
||||||
from bggpipe.normalize import normalize_title # noqa: E402
|
|
||||||
from bggpipe.resolve import _truncation_heads # noqa: E402
|
|
||||||
|
|
||||||
CIV_TITLE = (
|
CIV_TITLE = (
|
||||||
"CIVILIZATION Game of the Heroic Age - The Dawn of History 8000 BC to 250 BC"
|
"CIVILIZATION Game of the Heroic Age - The Dawn of History 8000 BC to 250 BC"
|
||||||
)
|
)
|
||||||
@@ -347,13 +350,15 @@ def test_run_resolve_saves_progress_when_token_missing(tmp_path):
|
|||||||
|
|
||||||
# -- post-resolve dedupe ------------------------------------------------
|
# -- post-resolve dedupe ------------------------------------------------
|
||||||
|
|
||||||
from bggpipe.bgg_client import cache_key as _cache_key # noqa: E402
|
|
||||||
from bggpipe.resolve import dedupe_matches # noqa: E402
|
|
||||||
|
|
||||||
|
|
||||||
def _mrow(
|
def _mrow(
|
||||||
title, bgg_id, photos, name="Joking Hazard",
|
title,
|
||||||
vstatus="version_unknown", vid="", status="auto",
|
bgg_id,
|
||||||
|
photos,
|
||||||
|
name="Joking Hazard",
|
||||||
|
vstatus="version_unknown",
|
||||||
|
vid="",
|
||||||
|
status="auto",
|
||||||
):
|
):
|
||||||
return {
|
return {
|
||||||
"title_raw": title,
|
"title_raw": title,
|
||||||
@@ -446,8 +451,6 @@ def test_dedupe_is_idempotent_and_skips_merged():
|
|||||||
|
|
||||||
|
|
||||||
def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
|
def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
|
||||||
from bggpipe.resolve import write_matches as _wm # noqa: F401
|
|
||||||
|
|
||||||
cache = tmp_path / "cache"
|
cache = tmp_path / "cache"
|
||||||
cache.mkdir()
|
cache.mkdir()
|
||||||
wingspan_xml = (
|
wingspan_xml = (
|
||||||
@@ -456,7 +459,7 @@ def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
|
|||||||
"</item></items>"
|
"</item></items>"
|
||||||
)
|
)
|
||||||
for query in ("Wingspan", "WINGSPAN!"):
|
for query in ("Wingspan", "WINGSPAN!"):
|
||||||
key = _cache_key(
|
key = cache_key(
|
||||||
"search", {"query": query, "type": "boardgame,boardgameexpansion"}
|
"search", {"query": query, "type": "boardgame,boardgameexpansion"}
|
||||||
)
|
)
|
||||||
(cache / key).write_text(wingspan_xml)
|
(cache / key).write_text(wingspan_xml)
|
||||||
@@ -482,3 +485,108 @@ def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
|
|||||||
assert saved["Wingspan"]["match_status"] == "auto"
|
assert saved["Wingspan"]["match_status"] == "auto"
|
||||||
assert saved["WINGSPAN!"]["match_status"] == "merged"
|
assert saved["WINGSPAN!"]["match_status"] == "merged"
|
||||||
assert saved["WINGSPAN!"]["merged_into"] == "Wingspan"
|
assert saved["WINGSPAN!"]["merged_into"] == "Wingspan"
|
||||||
|
|
||||||
|
|
||||||
|
# -- audit-fix regressions ----------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_resolve_force_rebuilds_from_scratch(client, tmp_path):
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
shutil.copy(TITLES_JSON, data_dir / "titles.json")
|
||||||
|
cfg = Config(data_dir=data_dir)
|
||||||
|
run_resolve(cfg, client=client)
|
||||||
|
|
||||||
|
# poison one row: force must throw it away and re-resolve everything
|
||||||
|
rows = read_matches(cfg.matches_path)
|
||||||
|
rows[0]["match_status"] = "rejected"
|
||||||
|
write_matches(cfg.matches_path, rows)
|
||||||
|
|
||||||
|
forced = run_resolve(cfg, force=True, client=client)
|
||||||
|
assert len(forced) == 7 # every title re-resolved, none skipped
|
||||||
|
fresh = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
|
||||||
|
assert fresh["Catan"]["match_status"] == "auto"
|
||||||
|
|
||||||
|
|
||||||
|
def test_new_photo_of_resolved_game_updates_row_instead_of_duplicating(
|
||||||
|
client, tmp_path
|
||||||
|
):
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
shutil.copy(TITLES_JSON, data_dir / "titles.json")
|
||||||
|
cfg = Config(data_dir=data_dir)
|
||||||
|
run_resolve(cfg, client=client)
|
||||||
|
n_rows = len(read_matches(cfg.matches_path))
|
||||||
|
|
||||||
|
# extract sees Catan again on a reshoot photo: the entry's photo set
|
||||||
|
# grows, its (title, photos) key changes
|
||||||
|
titles = json.loads((data_dir / "titles.json").read_text())
|
||||||
|
for entry in titles:
|
||||||
|
if entry["title_raw"] == "Catan":
|
||||||
|
entry["source_photos"] = sorted([*entry["source_photos"], "reshoot.jpg"])
|
||||||
|
(data_dir / "titles.json").write_text(json.dumps(titles))
|
||||||
|
|
||||||
|
assert run_resolve(cfg, client=client) == [] # nothing re-resolved
|
||||||
|
rows = read_matches(cfg.matches_path)
|
||||||
|
assert len(rows) == n_rows # and no duplicate row appended
|
||||||
|
catan = next(r for r in rows if r["title_raw"] == "Catan")
|
||||||
|
assert "reshoot.jpg" in catan["source_photos"] # provenance followed
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_never_overturns_a_human_veto():
|
||||||
|
a = _mrow("CATAN", "13", "p1.jpg")
|
||||||
|
b = _mrow("Catan", "13", "p2.jpg", status="approved")
|
||||||
|
b["dedupe_veto"] = "1" # review said: genuinely two copies
|
||||||
|
events = dedupe_matches([a, b], [])
|
||||||
|
assert events == []
|
||||||
|
assert b["match_status"] == "approved"
|
||||||
|
|
||||||
|
|
||||||
|
def test_publisher_pick_refuses_multiple_same_publisher_candidates():
|
||||||
|
from bggpipe.resolve import _publisher_pick
|
||||||
|
|
||||||
|
entry = TitleEntry(
|
||||||
|
title_raw="Sorcerer", title_normalized="sorcerer", publisher_hint="SPI"
|
||||||
|
)
|
||||||
|
cands = [
|
||||||
|
_cand(1, exact=True),
|
||||||
|
_cand(2, exact=True),
|
||||||
|
]
|
||||||
|
for c in cands:
|
||||||
|
c.publishers = ["Simulations Publications, Inc. (SPI)"]
|
||||||
|
assert _publisher_pick(entry, cands) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_publisher_pick_refuses_mixed_base_and_expansion():
|
||||||
|
from bggpipe.resolve import _publisher_pick
|
||||||
|
|
||||||
|
entry = TitleEntry(
|
||||||
|
title_raw="Wingspan", title_normalized="wingspan", publisher_hint="Stonemaier"
|
||||||
|
)
|
||||||
|
base = _cand(1, exact=True, type_="boardgame")
|
||||||
|
expansion = _cand(2, exact=True, type_="boardgameexpansion")
|
||||||
|
for c in (base, expansion):
|
||||||
|
c.publishers = ["Stonemaier Games"]
|
||||||
|
assert _publisher_pick(entry, [base, expansion]) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_version_handles_unknown_id():
|
||||||
|
from bggpipe.resolve import resolve_version
|
||||||
|
|
||||||
|
class EmptyThings:
|
||||||
|
def things(self, ids, **kwargs):
|
||||||
|
return []
|
||||||
|
|
||||||
|
entry = TitleEntry(title_raw="X", title_normalized="x", publisher_hint="Someone")
|
||||||
|
row = MatchRow(title_raw="X", bgg_id=999999)
|
||||||
|
resolve_version(EmptyThings(), entry, row) # must not raise
|
||||||
|
assert row.version_status == "version_unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_normalized_title_never_matches(client):
|
||||||
|
# 风声 normalizes to "" — empty-vs-empty must not count as exact
|
||||||
|
from bggpipe.resolve import _plausible_candidates
|
||||||
|
|
||||||
|
entry = TitleEntry(title_raw="风声", title_normalized="")
|
||||||
|
# any cached query works; candidates must be rejected regardless of name
|
||||||
|
assert _plausible_candidates(client, entry, "Catan") == []
|
||||||
|
|||||||
+95
-1
@@ -9,12 +9,13 @@ import json
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
import pytest
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGClient
|
from bggpipe.bgg_client import BGGClient
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.resolve import read_matches, write_matches
|
from bggpipe.resolve import read_matches, write_matches
|
||||||
from bggpipe.review import run_review
|
from bggpipe.review import ReviewSession, run_review
|
||||||
|
|
||||||
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
|
FIXTURES = Path(__file__).parent / "fixtures" / "bgg_cache"
|
||||||
|
|
||||||
@@ -288,3 +289,96 @@ def test_version_pass_is_skippable(tmp_path):
|
|||||||
)
|
)
|
||||||
(row,) = read_matches(cfg.matches_path)
|
(row,) = read_matches(cfg.matches_path)
|
||||||
assert row["version_status"] == "version_ambiguous" # untouched, review later
|
assert row["version_status"] == "version_ambiguous" # untouched, review later
|
||||||
|
|
||||||
|
|
||||||
|
# -- audit-fix regressions ----------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_veto_merge_persists_against_future_dedupe(tmp_path):
|
||||||
|
from bggpipe.resolve import read_matches
|
||||||
|
|
||||||
|
cfg = _setup(
|
||||||
|
tmp_path,
|
||||||
|
[
|
||||||
|
_row(title_raw="CATAN", match_status="merged", merged_into="Catan"),
|
||||||
|
_row(title_raw="Catan", match_status="auto", bgg_id="13"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
session = ReviewSession(
|
||||||
|
cfg,
|
||||||
|
console=quiet_console(),
|
||||||
|
input_fn=scripted(),
|
||||||
|
client=unauthorized_client(tmp_path),
|
||||||
|
)
|
||||||
|
merged = next(r for r in session.rows if r["match_status"] == "merged")
|
||||||
|
session.veto_merge(merged)
|
||||||
|
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
|
||||||
|
assert saved["CATAN"]["match_status"] == "approved"
|
||||||
|
assert saved["CATAN"]["dedupe_veto"] == "1" # survives resolve re-runs
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_save_never_leaves_memory_ahead_of_disk(tmp_path, monkeypatch):
|
||||||
|
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
|
||||||
|
session = ReviewSession(
|
||||||
|
cfg,
|
||||||
|
console=quiet_console(),
|
||||||
|
input_fn=scripted(),
|
||||||
|
client=unauthorized_client(tmp_path),
|
||||||
|
)
|
||||||
|
row = session.rows[0]
|
||||||
|
|
||||||
|
import bggpipe.review as review_mod
|
||||||
|
|
||||||
|
def exploding_write(path, rows):
|
||||||
|
raise OSError("disk full")
|
||||||
|
|
||||||
|
monkeypatch.setattr(review_mod, "write_matches", exploding_write)
|
||||||
|
with pytest.raises(OSError):
|
||||||
|
session.decide_reject(row)
|
||||||
|
# memory was rolled back to what disk actually holds
|
||||||
|
assert session.rows[0]["match_status"] == "unmatched"
|
||||||
|
assert session.decisions == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_merges_own_decision_over_concurrent_external_rewrite(tmp_path):
|
||||||
|
from bggpipe.resolve import read_matches, write_matches
|
||||||
|
|
||||||
|
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
|
||||||
|
session = ReviewSession(
|
||||||
|
cfg,
|
||||||
|
console=quiet_console(),
|
||||||
|
input_fn=scripted(),
|
||||||
|
client=unauthorized_client(tmp_path),
|
||||||
|
)
|
||||||
|
row = session.rows[0]
|
||||||
|
|
||||||
|
# resolve appends a new row in another terminal AFTER our session loaded
|
||||||
|
external = read_matches(cfg.matches_path)
|
||||||
|
external.append(_row(title_raw="Newcomer", match_status="auto", bgg_id="7"))
|
||||||
|
write_matches(cfg.matches_path, external)
|
||||||
|
|
||||||
|
session.decide_reject(row)
|
||||||
|
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
|
||||||
|
assert saved["Mystery"]["match_status"] == "rejected" # our decision
|
||||||
|
assert "Newcomer" in saved # their row survived too
|
||||||
|
|
||||||
|
|
||||||
|
def test_manual_id_unknown_to_bgg_warns_instead_of_crashing(tmp_path):
|
||||||
|
# an empty /thing result (mistyped id) used to crash the whole session
|
||||||
|
import httpx as _httpx
|
||||||
|
|
||||||
|
empty_things = BGGClient(
|
||||||
|
cache_dir=tmp_path / "cache",
|
||||||
|
transport=_httpx.MockTransport(
|
||||||
|
lambda req: _httpx.Response(200, text='<items total="0"></items>')
|
||||||
|
),
|
||||||
|
sleep=lambda s: None,
|
||||||
|
)
|
||||||
|
cfg = _setup(tmp_path, [_row(title_raw="Mystery", match_status="unmatched")])
|
||||||
|
session = ReviewSession(
|
||||||
|
cfg, console=quiet_console(), input_fn=scripted(), client=empty_things
|
||||||
|
)
|
||||||
|
session.decide_manual(session.rows[0], 999999)
|
||||||
|
assert session.rows[0]["match_status"] == "approved"
|
||||||
|
assert session.rows[0]["bgg_id"] == "999999"
|
||||||
|
assert any("no game with id 999999" in w for w in session.warnings)
|
||||||
|
|||||||
+88
-5
@@ -111,7 +111,7 @@ class FakeUploader:
|
|||||||
|
|
||||||
|
|
||||||
def test_build_queue_skips_logged_successes():
|
def test_build_queue_skips_logged_successes():
|
||||||
jobs, done, failed = build_queue(
|
jobs, done, failed, _ = build_queue(
|
||||||
[_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")],
|
[_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")],
|
||||||
[_update_row(collid="9")],
|
[_update_row(collid="9")],
|
||||||
[
|
[
|
||||||
@@ -127,7 +127,7 @@ def test_build_queue_skips_logged_successes():
|
|||||||
def test_build_queue_second_copy_is_a_distinct_job():
|
def test_build_queue_second_copy_is_a_distinct_job():
|
||||||
# Same game, different version: a separate physical copy, so a
|
# Same game, different version: a separate physical copy, so a
|
||||||
# logged add of one version must not swallow the other.
|
# logged add of one version must not swallow the other.
|
||||||
jobs, done, _ = build_queue(
|
jobs, done, _, _ = build_queue(
|
||||||
[
|
[
|
||||||
_add_row(bgg_id="1", version_id="10", version_name="First ed."),
|
_add_row(bgg_id="1", version_id="10", version_name="First ed."),
|
||||||
_add_row(bgg_id="1", version_id="11", version_name="Second ed."),
|
_add_row(bgg_id="1", version_id="11", version_name="Second ed."),
|
||||||
@@ -141,9 +141,11 @@ def test_build_queue_second_copy_is_a_distinct_job():
|
|||||||
|
|
||||||
def test_build_queue_failures_need_retry_flag():
|
def test_build_queue_failures_need_retry_flag():
|
||||||
log = [_log_row(action="add", bgg_id="1", status="failed")]
|
log = [_log_row(action="add", bgg_id="1", status="failed")]
|
||||||
jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log)
|
jobs, _, skipped, _ = build_queue([_add_row(bgg_id="1")], [], log)
|
||||||
assert jobs == [] and skipped == 1
|
assert jobs == [] and skipped == 1
|
||||||
jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
|
jobs, _, skipped, _ = build_queue(
|
||||||
|
[_add_row(bgg_id="1")], [], log, retry_failed=True
|
||||||
|
)
|
||||||
assert len(jobs) == 1 and skipped == 0
|
assert len(jobs) == 1 and skipped == 0
|
||||||
|
|
||||||
|
|
||||||
@@ -153,7 +155,7 @@ def test_build_queue_latest_log_entry_wins():
|
|||||||
_log_row(action="add", bgg_id="1", status="failed"),
|
_log_row(action="add", bgg_id="1", status="failed"),
|
||||||
_log_row(action="add", bgg_id="1", status="added"),
|
_log_row(action="add", bgg_id="1", status="added"),
|
||||||
]
|
]
|
||||||
jobs, done, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
|
jobs, done, _, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
|
||||||
assert jobs == [] and done == 1
|
assert jobs == [] and done == 1
|
||||||
|
|
||||||
|
|
||||||
@@ -300,3 +302,84 @@ def test_fresh_clone_marker_blocks_upload_without_cache_dir(tmp_path):
|
|||||||
_seed_data(tmp_path, to_add=[_add_row()])
|
_seed_data(tmp_path, to_add=[_add_row()])
|
||||||
with pytest.raises(typer.Exit):
|
with pytest.raises(typer.Exit):
|
||||||
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
|
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
# -- audit-fix regressions ----------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_error_aborts_without_poisoning_the_log(tmp_path):
|
||||||
|
from bggpipe.upload import LoginError
|
||||||
|
|
||||||
|
class BrokenLogin(FakeUploader):
|
||||||
|
def add_game(self, job):
|
||||||
|
raise LoginError("Cloudflare is challenging this browser")
|
||||||
|
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 4)])
|
||||||
|
results = run_upload(cfg, uploader=BrokenLogin(), sleep=lambda s: None, now=NOW)
|
||||||
|
assert results == [] # nothing logged: next run retries everything
|
||||||
|
assert not (tmp_path / "upload_log.csv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_three_identical_failures_abort_as_systemic(tmp_path):
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 6)])
|
||||||
|
fake = FakeUploader(failures={"Wingspan"}) # every job shares the name
|
||||||
|
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
|
||||||
|
assert len(results) == 3 # aborted after the third identical failure
|
||||||
|
logged = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
|
||||||
|
assert len(logged) == 3 # jobs 4-5 left unlogged and retryable
|
||||||
|
|
||||||
|
|
||||||
|
def test_added_no_version_is_done_and_verify_tolerates_it(tmp_path):
|
||||||
|
class NoVersionPicker(FakeUploader):
|
||||||
|
def add_game(self, job):
|
||||||
|
self.calls.append(job)
|
||||||
|
return "added_no_version", "version not in picker; added without version"
|
||||||
|
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(tmp_path, to_add=[_add_row(version_id="99", version_name="4th ed.")])
|
||||||
|
run_upload(cfg, uploader=NoVersionPicker(), sleep=lambda s: None, now=NOW)
|
||||||
|
# done: re-running must NOT re-add (a duplicate collection entry)
|
||||||
|
again = FakeUploader()
|
||||||
|
assert run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW) == []
|
||||||
|
assert again.calls == []
|
||||||
|
# verify: game present without the version is the EXPECTED outcome
|
||||||
|
log = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
|
||||||
|
assert verify_uploads(log, [_item(1, 10)]) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_to_add_csv_is_a_loud_precondition_failure(tmp_path):
|
||||||
|
cfg = _cfg(tmp_path) # no diff outputs seeded at all
|
||||||
|
with pytest.raises(typer.Exit):
|
||||||
|
run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
|
||||||
|
|
||||||
|
|
||||||
|
def test_second_update_for_same_game_is_deferred(tmp_path):
|
||||||
|
# the row-edit flow can't target a collid, so only one update per game
|
||||||
|
# per run is safe
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(
|
||||||
|
tmp_path,
|
||||||
|
to_update=[
|
||||||
|
_update_row(collid="9", bgg_id="2"),
|
||||||
|
_update_row(collid="10", bgg_id="2", vid="26", vname="2nd ed."),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
fake = FakeUploader()
|
||||||
|
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
|
||||||
|
assert [r["collid"] for r in results] == ["9"]
|
||||||
|
# after the first lands, the next run picks up the deferred one
|
||||||
|
again = FakeUploader()
|
||||||
|
results = run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW)
|
||||||
|
assert [j.collid for j in again.calls] == ["10"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_real_run_without_credentials_exits_before_any_browser(tmp_path, monkeypatch):
|
||||||
|
monkeypatch.delenv("BGG_USERNAME", raising=False)
|
||||||
|
monkeypatch.delenv("BGG_PASSWORD", raising=False)
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(tmp_path, to_add=[_add_row()])
|
||||||
|
with pytest.raises(typer.Exit):
|
||||||
|
run_upload(cfg, sleep=lambda s: None, now=NOW) # uploader=None: real path
|
||||||
|
assert not (tmp_path / "upload_log.csv").exists()
|
||||||
|
|||||||
@@ -383,3 +383,23 @@ def test_own_saves_do_not_count_as_external_changes(tmp_path):
|
|||||||
write_matches(cfg.matches_path, session.rows + [_row(title_raw="X")])
|
write_matches(cfg.matches_path, session.rows + [_row(title_raw="X")])
|
||||||
assert session.reload_if_changed() is True # someone else's
|
assert session.reload_if_changed() is True # someone else's
|
||||||
assert any(r["title_raw"] == "X" for r in session.rows)
|
assert any(r["title_raw"] == "X" for r in session.rows)
|
||||||
|
|
||||||
|
|
||||||
|
def test_session_warnings_surface_in_state(tmp_path):
|
||||||
|
web, cfg = make_client(tmp_path)
|
||||||
|
# a manual id triggers a lookup against the 401-ing client: the session
|
||||||
|
# degrades and the warning must reach the payload (the session console
|
||||||
|
# is a StringIO here — this is the only way the user ever sees it)
|
||||||
|
res = web.post(
|
||||||
|
"/api/decision",
|
||||||
|
json={
|
||||||
|
"title_raw": "Mystery",
|
||||||
|
"source_photos": "shelf.jpg",
|
||||||
|
"action": "manual",
|
||||||
|
"bgg_id": 42,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
assert res.status_code == 200
|
||||||
|
warnings = res.json()["warnings"]
|
||||||
|
assert any("couldn't look up id 42" in w for w in warnings)
|
||||||
|
assert warnings == web.get("/api/state").json()["warnings"]
|
||||||
|
|||||||
Reference in New Issue
Block a user