Re-audit round 2: 5 blind reviewers, 17 fixes, +12 tests
The re-run confirmed round 1 held and then caught second-order bugs in its own fixes plus two long-standing ones everyone missed. TUI decisions after a mid-session reload were counted but never written (rows are now re-adopted into the fresh list on every save, preferring undecided slots on duplicate keys); row_ix was computed by equality so duplicate rows shared an ordinal (identity now, merges included, veto sends it); upload job keys collided for two same-version copies (completions are counted per key, so --limit or an interrupt can no longer strand the second copy); diff consumes collids on exact-version matches (a vetoed same-version second copy was silently swallowed) and splits mismatches: report-only disagreement while an unclaimed copy exists, second-copy add only when every copy is claimed. Also: XML responses are validated and written atomically before caching (a torn or truncated 200 body can never poison a re-run), JSON artifacts write atomically, thing/search parsers refuse missing ids like the collection parser, empty game names are refused by the upload queue, a never-rendering version picker fails retryably instead of terminally, the systemic-failure abort compares exception types, blocked same-title entries defer as a group so positional pairing can't misalign, truncation heads pick the earliest separator, diff messages tell the truth when a token exists without a username, and the shared-constant sweep now actually covers every module (statuses, search types, marker names, client_for, ports). pydantic declared as a direct dependency. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -15,6 +15,7 @@ dependencies = [
|
|||||||
"fastapi>=0.141.1",
|
"fastapi>=0.141.1",
|
||||||
"uvicorn>=0.52.1",
|
"uvicorn>=0.52.1",
|
||||||
"playwright>=1.62.0",
|
"playwright>=1.62.0",
|
||||||
|
"pydantic>=2.13.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from bggpipe.config import STUB_CACHE_MARKER_NAME, STUB_DATA_MARKER_NAME
|
||||||
|
|
||||||
CACHE_MARKER_TEXT = (
|
CACHE_MARKER_TEXT = (
|
||||||
"This cache contains hand-written stub XML, not real BGG "
|
"This cache contains hand-written stub XML, not real BGG "
|
||||||
"responses. Data resolved from it must not be uploaded.\n"
|
"responses. Data resolved from it must not be uploaded.\n"
|
||||||
@@ -28,9 +30,9 @@ def esc(text: str) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def write_cache_marker(target: Path) -> None:
|
def write_cache_marker(target: Path) -> None:
|
||||||
(target / "STUB_FIXTURES.marker").write_text(CACHE_MARKER_TEXT)
|
(target / STUB_CACHE_MARKER_NAME).write_text(CACHE_MARKER_TEXT)
|
||||||
|
|
||||||
|
|
||||||
def write_data_marker(data_dir: Path = Path("data")) -> None:
|
def write_data_marker(data_dir: Path = Path("data")) -> None:
|
||||||
data_dir.mkdir(parents=True, exist_ok=True)
|
data_dir.mkdir(parents=True, exist_ok=True)
|
||||||
(data_dir / "STUB_DATA.marker").write_text(DATA_MARKER_TEXT)
|
(data_dir / STUB_DATA_MARKER_NAME).write_text(DATA_MARKER_TEXT)
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from pathlib import Path
|
|||||||
|
|
||||||
from fixture_common import esc, write_cache_marker, write_data_marker
|
from fixture_common import esc, write_cache_marker, write_data_marker
|
||||||
|
|
||||||
from bggpipe.bgg_client import cache_key
|
from bggpipe.bgg_client import SEARCH_TYPES, cache_key
|
||||||
|
|
||||||
TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
|
TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
|
||||||
|
|
||||||
@@ -27,7 +27,6 @@ TARGETS = (Path("tests/fixtures/bgg_cache"), Path("data/bgg_cache"))
|
|||||||
# gitignored, so this is what protects a fresh clone): upload refuses to
|
# gitignored, so this is what protects a fresh clone): upload refuses to
|
||||||
# run while it exists.
|
# run while it exists.
|
||||||
DATA_MARKER = Path("data/STUB_DATA.marker")
|
DATA_MARKER = Path("data/STUB_DATA.marker")
|
||||||
SEARCH_TYPES = "boardgame,boardgameexpansion"
|
|
||||||
|
|
||||||
BG, EXP = "boardgame", "boardgameexpansion"
|
BG, EXP = "boardgame", "boardgameexpansion"
|
||||||
|
|
||||||
@@ -345,7 +344,6 @@ def versions_xml(bgg_id: int, versions: list[tuple]) -> str:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
def main() -> None:
|
||||||
files: dict[str, str] = {}
|
files: dict[str, str] = {}
|
||||||
for query, results in SEARCHES.items():
|
for query, results in SEARCHES.items():
|
||||||
|
|||||||
@@ -16,10 +16,9 @@ from pathlib import Path
|
|||||||
|
|
||||||
from fixture_common import esc, write_cache_marker
|
from fixture_common import esc, write_cache_marker
|
||||||
|
|
||||||
from bggpipe.bgg_client import cache_key
|
from bggpipe.bgg_client import SEARCH_TYPES, cache_key
|
||||||
|
|
||||||
FIXTURE_CACHE = Path("tests/fixtures/bgg_cache")
|
FIXTURE_CACHE = Path("tests/fixtures/bgg_cache")
|
||||||
SEARCH_TYPES = "boardgame,boardgameexpansion"
|
|
||||||
|
|
||||||
|
|
||||||
def search_item(bgg_id: int, name: str, year: int | None, type_: str) -> str:
|
def search_item(bgg_id: int, name: str, year: int | None, type_: str) -> str:
|
||||||
|
|||||||
+13
-11
@@ -20,6 +20,7 @@ import httpx
|
|||||||
|
|
||||||
from bggpipe import __version__
|
from bggpipe import __version__
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
|
from bggpipe.fsio import atomic_write_text
|
||||||
from bggpipe.models import (
|
from bggpipe.models import (
|
||||||
BGGResponseError,
|
BGGResponseError,
|
||||||
CollectionItem,
|
CollectionItem,
|
||||||
@@ -29,9 +30,13 @@ from bggpipe.models import (
|
|||||||
parse_search,
|
parse_search,
|
||||||
parse_things,
|
parse_things,
|
||||||
parse_things_full,
|
parse_things_full,
|
||||||
|
validate_response,
|
||||||
)
|
)
|
||||||
|
|
||||||
BASE_URL = "https://boardgamegeek.com/xmlapi2"
|
BASE_URL = "https://boardgamegeek.com/xmlapi2"
|
||||||
|
# One home for the search-type filter: the fixture generators must build
|
||||||
|
# cache keys with the byte-identical string or every lookup silently misses.
|
||||||
|
SEARCH_TYPES = "boardgame,boardgameexpansion"
|
||||||
QUEUE_BACKOFF = (2.0, 5.0, 10.0, 30.0) # sleeps between the 5 attempts (spec)
|
QUEUE_BACKOFF = (2.0, 5.0, 10.0, 30.0) # sleeps between the 5 attempts (spec)
|
||||||
MAX_ATTEMPTS = 5
|
MAX_ATTEMPTS = 5
|
||||||
_UNSAFE = re.compile(r"[^A-Za-z0-9._=,-]+")
|
_UNSAFE = re.compile(r"[^A-Za-z0-9._=,-]+")
|
||||||
@@ -120,14 +125,13 @@ class BGGClient:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
if "<errors" in response.text[:120]:
|
try:
|
||||||
# BGG serves some errors as HTTP 200 <errors> XML (bad
|
# error documents AND malformed/truncated bodies must never
|
||||||
# username etc.) — caching one would poison every re-run
|
# reach the cache — they would poison every future run
|
||||||
raise BGGResponseError(
|
validate_response(response.text)
|
||||||
f"BGG error document for /{endpoint}: {response.text[:200]}"
|
except BGGResponseError as err:
|
||||||
)
|
raise BGGResponseError(f"/{endpoint}: {err}") from err
|
||||||
self.cache_dir.mkdir(parents=True, exist_ok=True)
|
atomic_write_text(cache_path, response.text)
|
||||||
cache_path.write_text(response.text)
|
|
||||||
return response.text
|
return response.text
|
||||||
|
|
||||||
raise BGGQueueTimeout(
|
raise BGGQueueTimeout(
|
||||||
@@ -138,9 +142,7 @@ class BGGClient:
|
|||||||
|
|
||||||
# -- typed endpoint wrappers ------------------------------------------
|
# -- typed endpoint wrappers ------------------------------------------
|
||||||
|
|
||||||
def search(
|
def search(self, query: str, types: str = SEARCH_TYPES) -> list[SearchResult]:
|
||||||
self, query: str, types: str = "boardgame,boardgameexpansion"
|
|
||||||
) -> list[SearchResult]:
|
|
||||||
return parse_search(self.get_xml("search", {"query": query, "type": types}))
|
return parse_search(self.get_xml("search", {"query": query, "type": types}))
|
||||||
|
|
||||||
def things(
|
def things(
|
||||||
|
|||||||
@@ -16,6 +16,10 @@ from pathlib import Path
|
|||||||
|
|
||||||
DEFAULT_CONFIG_PATH = Path("config.toml")
|
DEFAULT_CONFIG_PATH = Path("config.toml")
|
||||||
DEFAULT_REVIEW_PORT = 8377
|
DEFAULT_REVIEW_PORT = 8377
|
||||||
|
# Provenance marker filenames — the upload guard and both fixture
|
||||||
|
# generators must agree on these exactly.
|
||||||
|
STUB_CACHE_MARKER_NAME = "STUB_FIXTURES.marker"
|
||||||
|
STUB_DATA_MARKER_NAME = "STUB_DATA.marker"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -70,8 +74,8 @@ class Config:
|
|||||||
def stub_marker_paths(self) -> tuple[Path, Path]:
|
def stub_marker_paths(self) -> tuple[Path, Path]:
|
||||||
# gitignored (travels with the stub XML) + committed (guards clones)
|
# gitignored (travels with the stub XML) + committed (guards clones)
|
||||||
return (
|
return (
|
||||||
self.cache_dir / "STUB_FIXTURES.marker",
|
self.cache_dir / STUB_CACHE_MARKER_NAME,
|
||||||
self.data_dir / "STUB_DATA.marker",
|
self.data_dir / STUB_DATA_MARKER_NAME,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+111
-54
@@ -8,12 +8,12 @@ Two collection sources:
|
|||||||
logged-in-user exemption.
|
logged-in-user exemption.
|
||||||
|
|
||||||
Outputs both artifacts:
|
Outputs both artifacts:
|
||||||
- to_add.csv — recognized games not in the collection, including
|
- to_add.csv — recognized games not in the collection, plus additional
|
||||||
additional copies whose confident version matches no owned copy;
|
copies once every owned copy is claimed by another match row;
|
||||||
- to_update.csv — owned, VERSION-LESS entries where matching produced a
|
- to_update.csv — owned, VERSION-LESS entries where matching produced a
|
||||||
confident version (version_auto/version_approved). Strictly additive:
|
confident version. Strictly additive: entries that already carry a
|
||||||
entries that already carry a version are never touched — a further copy
|
version are never touched — a version mismatch against an unclaimed
|
||||||
with a different version becomes a to_add row instead.
|
copy is reported as a disagreement, nothing more.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -25,10 +25,11 @@ from pathlib import Path
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGClient, client_for
|
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.models import (
|
from bggpipe.models import (
|
||||||
CONFIDENT_VERSION_STATUSES,
|
CONFIDENT_VERSION_STATUSES,
|
||||||
|
RECOGNIZED_MATCH_STATUSES,
|
||||||
CollectionItem,
|
CollectionItem,
|
||||||
parse_collection,
|
parse_collection,
|
||||||
)
|
)
|
||||||
@@ -55,6 +56,7 @@ class DiffResult:
|
|||||||
to_update: list[dict] = field(default_factory=list)
|
to_update: list[dict] = field(default_factory=list)
|
||||||
already_owned: list[str] = field(default_factory=list) # title_raw
|
already_owned: list[str] = field(default_factory=list) # title_raw
|
||||||
second_copies: list[str] = field(default_factory=list) # notes for adds
|
second_copies: list[str] = field(default_factory=list) # notes for adds
|
||||||
|
disagreements: list[str] = field(default_factory=list) # report-only
|
||||||
unseen: list[CollectionItem] = field(default_factory=list)
|
unseen: list[CollectionItem] = field(default_factory=list)
|
||||||
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
|
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
|
||||||
rejected: int = 0
|
rejected: int = 0
|
||||||
@@ -99,6 +101,21 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
p for p in row["source_photos"].split(";") if p
|
p for p in row["source_photos"].split(";") if p
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def add_row(row: dict, confident: bool) -> dict:
|
||||||
|
photos = {p for p in row["source_photos"].split(";") if p}
|
||||||
|
photos |= merged_photos.get(row["title_raw"], set())
|
||||||
|
return {
|
||||||
|
"bgg_id": row["bgg_id"],
|
||||||
|
"bgg_name": row["bgg_name"],
|
||||||
|
"year": row["year"],
|
||||||
|
"type": row["type"],
|
||||||
|
"version_id": row["version_id"] if confident else "",
|
||||||
|
"version_name": row["version_name"] if confident else "",
|
||||||
|
"title_raw": row["title_raw"],
|
||||||
|
"source_photos": ";".join(sorted(photos)),
|
||||||
|
}
|
||||||
|
|
||||||
|
recognized: list[dict] = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
status = row["match_status"]
|
status = row["match_status"]
|
||||||
if status == "rejected":
|
if status == "rejected":
|
||||||
@@ -107,53 +124,43 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
if status == "merged":
|
if status == "merged":
|
||||||
result.merged += 1 # represented by its survivor row
|
result.merged += 1 # represented by its survivor row
|
||||||
continue
|
continue
|
||||||
if status not in ("auto", "approved") or not row["bgg_id"]:
|
if status not in RECOGNIZED_MATCH_STATUSES or not row["bgg_id"]:
|
||||||
result.pending.append(row["title_raw"])
|
result.pending.append(row["title_raw"])
|
||||||
continue
|
continue
|
||||||
|
|
||||||
result.recognized += 1
|
result.recognized += 1
|
||||||
bgg_id = int(row["bgg_id"])
|
recognized.append(row)
|
||||||
copies = by_object.get(bgg_id, [])
|
if by_object.get(int(row["bgg_id"])):
|
||||||
if copies:
|
seen_object_ids.add(int(row["bgg_id"]))
|
||||||
seen_object_ids.add(bgg_id)
|
|
||||||
|
|
||||||
confident = (
|
def unconsumed(bgg_id: int) -> list[CollectionItem]:
|
||||||
|
return [
|
||||||
|
c for c in by_object.get(bgg_id, []) if c.coll_id not in consumed_collids
|
||||||
|
]
|
||||||
|
|
||||||
|
def is_confident(row: dict) -> bool:
|
||||||
|
return bool(
|
||||||
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
|
row["version_status"] in CONFIDENT_VERSION_STATUSES and row["version_id"]
|
||||||
)
|
)
|
||||||
version_id = int(row["version_id"]) if confident else None
|
|
||||||
|
|
||||||
def add_row(row: dict = row, confident: bool = confident) -> dict:
|
# Pass 1 — confident-version rows claim copies first (an exact version
|
||||||
photos = {p for p in row["source_photos"].split(";") if p}
|
# match, then a versionless copy to upgrade). Bare rows must not steal
|
||||||
photos |= merged_photos.get(row["title_raw"], set())
|
# a versionless copy a confident row would have upgraded.
|
||||||
return {
|
for row in (r for r in recognized if is_confident(r)):
|
||||||
"bgg_id": row["bgg_id"],
|
bgg_id = int(row["bgg_id"])
|
||||||
"bgg_name": row["bgg_name"],
|
version_id = int(row["version_id"])
|
||||||
"year": row["year"],
|
remaining = unconsumed(bgg_id)
|
||||||
"type": row["type"],
|
if not by_object.get(bgg_id):
|
||||||
"version_id": row["version_id"] if confident else "",
|
result.to_add.append(add_row(row, True))
|
||||||
"version_name": row["version_name"] if confident else "",
|
|
||||||
"title_raw": row["title_raw"],
|
|
||||||
"source_photos": ";".join(sorted(photos)),
|
|
||||||
}
|
|
||||||
|
|
||||||
if not copies:
|
|
||||||
result.to_add.append(add_row())
|
|
||||||
continue
|
continue
|
||||||
|
matching = [c for c in remaining if c.version_id == version_id]
|
||||||
if not confident:
|
if matching:
|
||||||
# bare id with unknown version: owned if any copy exists
|
# exact (bgg_id, version) pair: consume, so a SECOND row with
|
||||||
|
# the same version (a vetoed duplicate = a real second copy)
|
||||||
|
# falls through to the branches below instead of vanishing
|
||||||
|
consumed_collids.add(matching[0].coll_id)
|
||||||
result.already_owned.append(row["title_raw"])
|
result.already_owned.append(row["title_raw"])
|
||||||
continue
|
continue
|
||||||
|
versionless = [c for c in remaining if c.version_id is None]
|
||||||
if any(c.version_id == version_id for c in copies):
|
|
||||||
result.already_owned.append(row["title_raw"])
|
|
||||||
continue
|
|
||||||
|
|
||||||
versionless = [
|
|
||||||
c
|
|
||||||
for c in copies
|
|
||||||
if c.version_id is None and c.coll_id not in consumed_collids
|
|
||||||
]
|
|
||||||
if versionless:
|
if versionless:
|
||||||
target = versionless[0]
|
target = versionless[0]
|
||||||
consumed_collids.add(target.coll_id)
|
consumed_collids.add(target.coll_id)
|
||||||
@@ -167,18 +174,49 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
"version_name": row["version_name"],
|
"version_name": row["version_name"],
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
elif remaining:
|
||||||
|
# an unclaimed copy exists but carries a DIFFERENT version:
|
||||||
|
# most likely the same physical box mis-scored — report, never
|
||||||
|
# touch, never duplicate (spec: report the disagreement)
|
||||||
|
consumed_collids.add(remaining[0].coll_id)
|
||||||
|
result.already_owned.append(row["title_raw"])
|
||||||
|
result.disagreements.append(
|
||||||
|
f"{row['title_raw']}: photo suggests version "
|
||||||
|
f"{row['version_name']!r} ({row['version_id']}) but the "
|
||||||
|
"remaining collection entry carries a different version — "
|
||||||
|
"left untouched"
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
# no remaining copy can take this version: every copy either
|
# every copy is claimed by other match rows: this row is an
|
||||||
# already carries a different version or was consumed by another
|
# additional physical copy (spec: a pair is owned only when a
|
||||||
# match — per spec this is an additional physical copy to ADD
|
# collection item matches both ids)
|
||||||
# (existing entries are never touched)
|
result.to_add.append(add_row(row, True))
|
||||||
result.to_add.append(add_row())
|
|
||||||
result.second_copies.append(
|
result.second_copies.append(
|
||||||
f"{row['title_raw']}: adding as a NEW copy with version "
|
f"{row['title_raw']}: adding as a NEW copy with version "
|
||||||
f"{row['version_name']!r} ({row['version_id']}) — every "
|
f"{row['version_name']!r} ({row['version_id']}) — every "
|
||||||
"existing entry of this game keeps its current version"
|
"existing entry of this game keeps its current version"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Pass 2 — bare (version-unknown) rows: owned while unclaimed copies
|
||||||
|
# remain; extras beyond the owned count (vetoed duplicates) are added
|
||||||
|
# as version-less new entries.
|
||||||
|
for row in (r for r in recognized if not is_confident(r)):
|
||||||
|
bgg_id = int(row["bgg_id"])
|
||||||
|
if not by_object.get(bgg_id):
|
||||||
|
result.to_add.append(add_row(row, False))
|
||||||
|
continue
|
||||||
|
remaining = unconsumed(bgg_id)
|
||||||
|
if remaining:
|
||||||
|
consumed_collids.add(remaining[0].coll_id)
|
||||||
|
result.already_owned.append(row["title_raw"])
|
||||||
|
else:
|
||||||
|
result.to_add.append(add_row(row, False))
|
||||||
|
result.second_copies.append(
|
||||||
|
f"{row['title_raw']}: adding as a NEW version-less copy — "
|
||||||
|
"every existing entry of this game is claimed by another "
|
||||||
|
"match row"
|
||||||
|
)
|
||||||
|
|
||||||
result.unseen = [
|
result.unseen = [
|
||||||
item for item in collection if item.object_id not in seen_object_ids
|
item for item in collection if item.object_id not in seen_object_ids
|
||||||
]
|
]
|
||||||
@@ -201,15 +239,30 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
|||||||
typer.echo(f"{cfg.matches_path} is empty — run `bggpipe resolve` first.")
|
typer.echo(f"{cfg.matches_path} is empty — run `bggpipe resolve` first.")
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
|
|
||||||
if os.environ.get("BGG_API_TOKEN") and cfg.bgg_username:
|
has_token = bool(os.environ.get("BGG_API_TOKEN"))
|
||||||
|
if has_token and cfg.bgg_username:
|
||||||
typer.echo("Fetching live collection from BGG…")
|
typer.echo("Fetching live collection from BGG…")
|
||||||
client = client or client_for(cfg)
|
client = client or client_for(cfg)
|
||||||
collection = client.collection_full(cfg.bgg_username, refresh=True)
|
try:
|
||||||
|
collection = client.collection_full(cfg.bgg_username, refresh=True)
|
||||||
|
except BGGAuthError as err:
|
||||||
|
# a present-but-invalid token must not traceback when the
|
||||||
|
# snapshot fallback is sitting right there
|
||||||
|
typer.echo(f"Live fetch failed ({err}) — using snapshot files.")
|
||||||
|
collection = load_snapshot_collection(cfg.data_dir)
|
||||||
else:
|
else:
|
||||||
typer.echo(
|
if has_token:
|
||||||
"No BGG_API_TOKEN — using collection snapshot files in "
|
# saying "No BGG_API_TOKEN" here would be false and misdirect
|
||||||
f"{cfg.data_dir}/ (live mode takes over once the token exists)."
|
# the user's debugging — the missing half is the username
|
||||||
)
|
typer.echo(
|
||||||
|
"BGG_API_TOKEN is set but BGG_USERNAME is not (is .env "
|
||||||
|
"loaded?) — using snapshot files instead of live mode."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
typer.echo(
|
||||||
|
"No BGG_API_TOKEN — using collection snapshot files in "
|
||||||
|
f"{cfg.data_dir}/ (live mode takes over once the token exists)."
|
||||||
|
)
|
||||||
collection = load_snapshot_collection(cfg.data_dir)
|
collection = load_snapshot_collection(cfg.data_dir)
|
||||||
|
|
||||||
result = compute_diff(rows, collection)
|
result = compute_diff(rows, collection)
|
||||||
@@ -228,6 +281,10 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
|||||||
typer.echo("\nSecond copies (verify these on the dry run before upload):")
|
typer.echo("\nSecond copies (verify these on the dry run before upload):")
|
||||||
for line in result.second_copies:
|
for line in result.second_copies:
|
||||||
typer.echo(f" - {line}")
|
typer.echo(f" - {line}")
|
||||||
|
if result.disagreements:
|
||||||
|
typer.echo("\nVersion disagreements (reported only — nothing changed):")
|
||||||
|
for line in result.disagreements:
|
||||||
|
typer.echo(f" - {line}")
|
||||||
if result.pending:
|
if result.pending:
|
||||||
typer.echo("\nStill pending review: " + ", ".join(result.pending))
|
typer.echo("\nStill pending review: " + ", ".join(result.pending))
|
||||||
if result.unseen:
|
if result.unseen:
|
||||||
|
|||||||
+12
-6
@@ -19,16 +19,20 @@ import json
|
|||||||
|
|
||||||
import typer
|
import typer
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGAuthError, BGGClient
|
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
|
from bggpipe.fsio import atomic_write_text
|
||||||
|
from bggpipe.models import (
|
||||||
|
CONFIDENT_VERSION_STATUSES,
|
||||||
|
RECOGNIZED_MATCH_STATUSES,
|
||||||
|
)
|
||||||
from bggpipe.resolve import read_matches
|
from bggpipe.resolve import read_matches
|
||||||
|
|
||||||
BATCH_SIZE = 20
|
BATCH_SIZE = 20
|
||||||
_CONFIDENT_VERSION = ("version_auto", "version_approved")
|
|
||||||
|
|
||||||
|
|
||||||
def _version_info(row: dict) -> dict | None:
|
def _version_info(row: dict) -> dict | None:
|
||||||
if row["version_status"] not in _CONFIDENT_VERSION or not row["version_id"]:
|
if row["version_status"] not in CONFIDENT_VERSION_STATUSES or not row["version_id"]:
|
||||||
return None
|
return None
|
||||||
version_id = int(row["version_id"])
|
version_id = int(row["version_id"])
|
||||||
for cand in json.loads(row["version_candidates_json"] or "[]"):
|
for cand in json.loads(row["version_candidates_json"] or "[]"):
|
||||||
@@ -60,7 +64,7 @@ def run_enrich(
|
|||||||
|
|
||||||
targets: list[tuple[str, int, dict | None]] = []
|
targets: list[tuple[str, int, dict | None]] = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
|
if row["match_status"] not in RECOGNIZED_MATCH_STATUSES or not row["bgg_id"]:
|
||||||
continue
|
continue
|
||||||
version = _version_info(row)
|
version = _version_info(row)
|
||||||
key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
|
key = f"{row['bgg_id']}:{version['version_id']}" if version else row["bgg_id"]
|
||||||
@@ -70,7 +74,7 @@ def run_enrich(
|
|||||||
games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
|
games: dict = json.loads(games_path.read_text()) if games_path.exists() else {}
|
||||||
|
|
||||||
need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
|
need = sorted({bgg_id for key, bgg_id, _ in targets if refresh or key not in games})
|
||||||
client = client or BGGClient(cfg.cache_dir, cfg.rate_limit_seconds)
|
client = client or client_for(cfg)
|
||||||
|
|
||||||
fetched: dict[int, dict] = {}
|
fetched: dict[int, dict] = {}
|
||||||
blocked = False
|
blocked = False
|
||||||
@@ -90,7 +94,9 @@ def run_enrich(
|
|||||||
updated += 1
|
updated += 1
|
||||||
|
|
||||||
games_path.parent.mkdir(parents=True, exist_ok=True)
|
games_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
games_path.write_text(json.dumps(games, indent=2, ensure_ascii=False) + "\n")
|
atomic_write_text(
|
||||||
|
games_path, json.dumps(games, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
|
|
||||||
typer.echo(
|
typer.echo(
|
||||||
f"games.json: {len(games)} entr{'y' if len(games) == 1 else 'ies'} "
|
f"games.json: {len(games)} entr{'y' if len(games) == 1 else 'ies'} "
|
||||||
|
|||||||
+11
-5
@@ -19,6 +19,7 @@ from pathlib import Path
|
|||||||
import typer
|
import typer
|
||||||
|
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
|
from bggpipe.fsio import atomic_write_text
|
||||||
from bggpipe.normalize import normalize_title
|
from bggpipe.normalize import normalize_title
|
||||||
|
|
||||||
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".heic"}
|
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".heic"}
|
||||||
@@ -105,7 +106,8 @@ def prepare_image(path: Path) -> tuple[str, str]:
|
|||||||
|
|
||||||
def parse_vision_response(text: str) -> tuple[list[dict], list[dict], int]:
|
def parse_vision_response(text: str) -> tuple[list[dict], list[dict], int]:
|
||||||
"""Parse the model's JSON defensively: strip code fences, locate the
|
"""Parse the model's JSON defensively: strip code fences, locate the
|
||||||
payload amid any prose. Returns (title entries, unidentified sightings).
|
payload amid any prose. Returns (title entries, unidentified sightings,
|
||||||
|
dropped-malformed-entry count).
|
||||||
A bare JSON array (the pre-unidentified response shape) still parses —
|
A bare JSON array (the pre-unidentified response shape) still parses —
|
||||||
it's all titles."""
|
it's all titles."""
|
||||||
cleaned = _CODE_FENCE.sub("", text).strip()
|
cleaned = _CODE_FENCE.sub("", text).strip()
|
||||||
@@ -268,9 +270,11 @@ def rebuild_artifacts(
|
|||||||
unidentified[photo] = data["unidentified"]
|
unidentified[photo] = data["unidentified"]
|
||||||
deduped = dedupe_entries(entries)
|
deduped = dedupe_entries(entries)
|
||||||
titles_path.parent.mkdir(parents=True, exist_ok=True)
|
titles_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
titles_path.write_text(json.dumps(deduped, indent=2, ensure_ascii=False) + "\n")
|
atomic_write_text(
|
||||||
unidentified_path.write_text(
|
titles_path, json.dumps(deduped, indent=2, ensure_ascii=False) + "\n"
|
||||||
json.dumps(unidentified, indent=2, ensure_ascii=False) + "\n"
|
)
|
||||||
|
atomic_write_text(
|
||||||
|
unidentified_path, json.dumps(unidentified, indent=2, ensure_ascii=False) + "\n"
|
||||||
)
|
)
|
||||||
return deduped, unidentified
|
return deduped, unidentified
|
||||||
|
|
||||||
@@ -311,7 +315,9 @@ def run_extract(
|
|||||||
typer.echo(f" {photo.name}: already extracted, skipping")
|
typer.echo(f" {photo.name}: already extracted, skipping")
|
||||||
continue
|
continue
|
||||||
result = extract_photo(photo, vision)
|
result = extract_photo(photo, vision)
|
||||||
raw_path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n")
|
atomic_write_text(
|
||||||
|
raw_path, json.dumps(result, indent=2, ensure_ascii=False) + "\n"
|
||||||
|
)
|
||||||
note = (
|
note = (
|
||||||
f" ({len(result['unidentified'])} unidentified)"
|
f" ({len(result['unidentified'])} unidentified)"
|
||||||
if result["unidentified"]
|
if result["unidentified"]
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
"""Atomic file writes for every accumulated artifact.
|
||||||
|
|
||||||
|
A kill mid-write must never leave a torn file that poisons future runs —
|
||||||
|
the same tmp + os.replace guarantee write_matches gives matches.csv,
|
||||||
|
available to JSON artifacts and the XML response cache.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def atomic_write_text(path: Path, text: str) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = path.with_name(path.name + ".tmp")
|
||||||
|
tmp.write_text(text)
|
||||||
|
os.replace(tmp, path)
|
||||||
+30
-8
@@ -12,10 +12,11 @@ class BGGResponseError(Exception):
|
|||||||
"""The API returned a well-formed error document (e.g. bad username)."""
|
"""The API returned a well-formed error document (e.g. bad username)."""
|
||||||
|
|
||||||
|
|
||||||
# The one status predicate the whole pipeline shares: a version is trusted
|
# The two status predicates the whole pipeline shares: a version is trusted
|
||||||
# for diff/upload/enrich only when matching produced it confidently or a
|
# for diff/upload/enrich only when matching produced it confidently or a
|
||||||
# human approved it.
|
# human approved it, and a row reaches diff/enrich only when its match did.
|
||||||
CONFIDENT_VERSION_STATUSES = ("version_auto", "version_approved")
|
CONFIDENT_VERSION_STATUSES = ("version_auto", "version_approved")
|
||||||
|
RECOGNIZED_MATCH_STATUSES = ("auto", "approved")
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -78,9 +79,11 @@ def _attr_int(elem: ET.Element | None, attr: str = "value") -> int | None:
|
|||||||
|
|
||||||
def parse_search(xml_text: str) -> list[SearchResult]:
|
def parse_search(xml_text: str) -> list[SearchResult]:
|
||||||
results = []
|
results = []
|
||||||
|
skipped = 0
|
||||||
for item in _root(xml_text).findall("item"):
|
for item in _root(xml_text).findall("item"):
|
||||||
name_elem = item.find("name")
|
name_elem = item.find("name")
|
||||||
if name_elem is None or item.get("id") is None:
|
if name_elem is None or item.get("id") is None:
|
||||||
|
skipped += 1 # tolerate stragglers; wholesale drift raises below
|
||||||
continue
|
continue
|
||||||
results.append(
|
results.append(
|
||||||
SearchResult(
|
SearchResult(
|
||||||
@@ -91,6 +94,11 @@ def parse_search(xml_text: str) -> list[SearchResult]:
|
|||||||
type=item.get("type", "boardgame"),
|
type=item.get("type", "boardgame"),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if skipped and not results:
|
||||||
|
raise BGGResponseError(
|
||||||
|
f"search response had {skipped} item(s), none parseable — "
|
||||||
|
"schema drift? Bad data must not look like no results."
|
||||||
|
)
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
@@ -125,7 +133,7 @@ def parse_things(xml_text: str) -> list[ThingDetails]:
|
|||||||
]
|
]
|
||||||
things.append(
|
things.append(
|
||||||
ThingDetails(
|
ThingDetails(
|
||||||
bgg_id=int(item.get("id", 0)),
|
bgg_id=int(_required_attr(item, "id")),
|
||||||
name=name.get("value", "") if name is not None else "",
|
name=name.get("value", "") if name is not None else "",
|
||||||
year=_attr_int(item.find("yearpublished")),
|
year=_attr_int(item.find("yearpublished")),
|
||||||
type=item.get("type", "boardgame"),
|
type=item.get("type", "boardgame"),
|
||||||
@@ -181,7 +189,7 @@ def parse_things_full(xml_text: str) -> list[dict]:
|
|||||||
best_player_counts.append(results.get("numplayers"))
|
best_player_counts.append(results.get("numplayers"))
|
||||||
games.append(
|
games.append(
|
||||||
{
|
{
|
||||||
"bgg_id": int(item.get("id", 0)),
|
"bgg_id": int(_required_attr(item, "id")),
|
||||||
"type": item.get("type", "boardgame"),
|
"type": item.get("type", "boardgame"),
|
||||||
"name": name.get("value", "") if name is not None else "",
|
"name": name.get("value", "") if name is not None else "",
|
||||||
"year": _attr_int(item.find("yearpublished")),
|
"year": _attr_int(item.find("yearpublished")),
|
||||||
@@ -215,16 +223,26 @@ def parse_things_full(xml_text: str) -> list[dict]:
|
|||||||
return games
|
return games
|
||||||
|
|
||||||
|
|
||||||
def _required_attr(item, name: str) -> str:
|
def _required_attr(item: ET.Element, name: str) -> str:
|
||||||
value = item.get(name)
|
value = item.get(name)
|
||||||
if not value:
|
if not value:
|
||||||
raise BGGResponseError(
|
raise BGGResponseError(
|
||||||
f"collection item missing {name!r} — truncated or unexpected "
|
f"response item missing {name!r} — truncated or unexpected "
|
||||||
"response; refusing to feed it to the diff"
|
"response; refusing to coerce a missing id"
|
||||||
)
|
)
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def validate_response(xml_text: str) -> None:
|
||||||
|
"""Raise BGGResponseError for error documents AND malformed XML — the
|
||||||
|
client calls this before caching, so a torn or truncated 200 body can
|
||||||
|
never poison the cache."""
|
||||||
|
try:
|
||||||
|
_root(xml_text)
|
||||||
|
except ET.ParseError as err:
|
||||||
|
raise BGGResponseError(f"malformed XML: {err}") from err
|
||||||
|
|
||||||
|
|
||||||
def parse_collection(xml_text: str) -> list[CollectionItem]:
|
def parse_collection(xml_text: str) -> list[CollectionItem]:
|
||||||
items = []
|
items = []
|
||||||
for item in _root(xml_text).findall("item"):
|
for item in _root(xml_text).findall("item"):
|
||||||
@@ -240,7 +258,11 @@ def parse_collection(xml_text: str) -> list[CollectionItem]:
|
|||||||
name=item.findtext("name", default=""),
|
name=item.findtext("name", default=""),
|
||||||
subtype=item.get("subtype", "boardgame"),
|
subtype=item.get("subtype", "boardgame"),
|
||||||
own=status is not None and status.get("own") == "1",
|
own=status is not None and status.get("own") == "1",
|
||||||
year=int(year_text) if year_text and year_text.isdigit() else None,
|
year=(
|
||||||
|
int(year_text)
|
||||||
|
if year_text and year_text.lstrip("-").isdigit()
|
||||||
|
else None
|
||||||
|
),
|
||||||
version_id=(
|
version_id=(
|
||||||
int(version_item.get("id"))
|
int(version_item.get("id"))
|
||||||
if version_item is not None and version_item.get("id")
|
if version_item is not None and version_item.get("id")
|
||||||
|
|||||||
+27
-11
@@ -22,7 +22,11 @@ from rapidfuzz import fuzz
|
|||||||
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
|
from bggpipe.bgg_client import BGGAuthError, BGGClient, client_for
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
from bggpipe.extract import cues_conflict
|
from bggpipe.extract import cues_conflict
|
||||||
from bggpipe.models import CONFIDENT_VERSION_STATUSES, GameVersion
|
from bggpipe.models import (
|
||||||
|
CONFIDENT_VERSION_STATUSES,
|
||||||
|
RECOGNIZED_MATCH_STATUSES,
|
||||||
|
GameVersion,
|
||||||
|
)
|
||||||
from bggpipe.normalize import normalize_title
|
from bggpipe.normalize import normalize_title
|
||||||
|
|
||||||
FUZZY_THRESHOLD = 90
|
FUZZY_THRESHOLD = 90
|
||||||
@@ -149,8 +153,10 @@ def load_titles(path: Path) -> list[TitleEntry]:
|
|||||||
entries.append(
|
entries.append(
|
||||||
TitleEntry(
|
TitleEntry(
|
||||||
title_raw=title_raw,
|
title_raw=title_raw,
|
||||||
title_normalized=raw.get("title_normalized")
|
# always recompute: a stale/hand-written stored value would
|
||||||
or normalize_title(title_raw),
|
# silently break exact matching (both sides must normalize
|
||||||
|
# by the CURRENT rules)
|
||||||
|
title_normalized=normalize_title(title_raw),
|
||||||
confidence=raw.get("confidence", "high"),
|
confidence=raw.get("confidence", "high"),
|
||||||
publisher_hint=raw.get("publisher_hint") or "",
|
publisher_hint=raw.get("publisher_hint") or "",
|
||||||
edition_hint=raw.get("edition_hint") or "",
|
edition_hint=raw.get("edition_hint") or "",
|
||||||
@@ -173,9 +179,10 @@ def _truncation_heads(title_raw: str) -> list[str]:
|
|||||||
before the first subtitle separator, before a "(The) Game ..."
|
before the first subtitle separator, before a "(The) Game ..."
|
||||||
descriptor, then the first two words as a last resort."""
|
descriptor, then the first two words as a last resort."""
|
||||||
heads: list[str] = []
|
heads: list[str] = []
|
||||||
sep_head = next(
|
present = [(title_raw.find(sep), sep) for sep in _SEPARATORS if sep in title_raw]
|
||||||
(title_raw.split(sep)[0] for sep in _SEPARATORS if sep in title_raw), None
|
# earliest separator wins — priority order would let " - " late in the
|
||||||
)
|
# title beat an early ": ", yielding heads like "Blorvath: Quest"
|
||||||
|
sep_head = title_raw.split(min(present)[1])[0] if present else None
|
||||||
if sep_head:
|
if sep_head:
|
||||||
heads.append(sep_head)
|
heads.append(sep_head)
|
||||||
match = _GAME_WORD.search(title_raw)
|
match = _GAME_WORD.search(title_raw)
|
||||||
@@ -203,13 +210,14 @@ def _plausible_candidates(
|
|||||||
"""Search BGG and keep plausible candidates, one per id: exact-normalized
|
"""Search BGG and keep plausible candidates, one per id: exact-normalized
|
||||||
or fuzzy>=90 against the FULL title, or — on truncated retries — exact
|
or fuzzy>=90 against the FULL title, or — on truncated retries — exact
|
||||||
(only exact: truncation must stay conservative) against the head."""
|
(only exact: truncation must stay conservative) against the head."""
|
||||||
|
# a fully non-Latin title normalizes to "" — empty-vs-empty is not a
|
||||||
|
# match (token_sort_ratio("", "") is 100), and searching would only
|
||||||
|
# spend rate-limited requests to prove nothing
|
||||||
|
if not entry.title_normalized:
|
||||||
|
return []
|
||||||
by_id: dict[int, Candidate] = {}
|
by_id: dict[int, Candidate] = {}
|
||||||
for result in client.search(query):
|
for result in client.search(query):
|
||||||
norm = normalize_title(result.name)
|
norm = normalize_title(result.name)
|
||||||
# a fully non-Latin title normalizes to "" — empty-vs-empty is not a
|
|
||||||
# match (and token_sort_ratio("", "") is 100, so guard fuzzy too)
|
|
||||||
if not entry.title_normalized:
|
|
||||||
continue
|
|
||||||
exact = norm == entry.title_normalized
|
exact = norm == entry.title_normalized
|
||||||
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
|
fuzzy = fuzz.token_sort_ratio(norm, entry.title_normalized)
|
||||||
if not exact and fuzzy < FUZZY_THRESHOLD:
|
if not exact and fuzzy < FUZZY_THRESHOLD:
|
||||||
@@ -427,7 +435,7 @@ def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEven
|
|||||||
|
|
||||||
groups: dict[tuple[str, str], list[dict]] = {}
|
groups: dict[tuple[str, str], list[dict]] = {}
|
||||||
for row in rows:
|
for row in rows:
|
||||||
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
|
if row["match_status"] not in RECOGNIZED_MATCH_STATUSES or not row["bgg_id"]:
|
||||||
continue
|
continue
|
||||||
if row.get("dedupe_veto"):
|
if row.get("dedupe_veto"):
|
||||||
# a human already ruled "this is a genuinely separate copy" —
|
# a human already ruled "this is a genuinely separate copy" —
|
||||||
@@ -516,10 +524,17 @@ def run_resolve(
|
|||||||
skipped = 0
|
skipped = 0
|
||||||
photos_updated = False
|
photos_updated = False
|
||||||
blocked: list[str] = []
|
blocked: list[str] = []
|
||||||
|
blocked_titles: set[str] = set()
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
ix = seen_per_title.get(entry.title_raw, 0)
|
ix = seen_per_title.get(entry.title_raw, 0)
|
||||||
seen_per_title[entry.title_raw] = ix + 1
|
seen_per_title[entry.title_raw] = ix + 1
|
||||||
paired = rows_by_title.get(entry.title_raw, [])
|
paired = rows_by_title.get(entry.title_raw, [])
|
||||||
|
if entry.title_raw in blocked_titles and ix >= len(paired):
|
||||||
|
# an earlier same-title entry is waiting on the token: resolving
|
||||||
|
# this one now would append a row at the wrong position and
|
||||||
|
# corrupt next run's positional pairing — defer the whole group
|
||||||
|
blocked.append(entry.title_raw)
|
||||||
|
continue
|
||||||
if ix < len(paired):
|
if ix < len(paired):
|
||||||
row_dict = paired[ix]
|
row_dict = paired[ix]
|
||||||
photos = ";".join(entry.source_photos)
|
photos = ";".join(entry.source_photos)
|
||||||
@@ -534,6 +549,7 @@ def run_resolve(
|
|||||||
# No API token yet: cached titles still resolve; the rest wait.
|
# No API token yet: cached titles still resolve; the rest wait.
|
||||||
# No row is written, so a future run picks them up untouched.
|
# No row is written, so a future run picks them up untouched.
|
||||||
blocked.append(entry.title_raw)
|
blocked.append(entry.title_raw)
|
||||||
|
blocked_titles.add(entry.title_raw)
|
||||||
typer.echo(f" {entry.title_raw!r} -> waiting on BGG API token")
|
typer.echo(f" {entry.title_raw!r} -> waiting on BGG API token")
|
||||||
continue
|
continue
|
||||||
new_rows.append(row)
|
new_rows.append(row)
|
||||||
|
|||||||
+35
-16
@@ -110,23 +110,42 @@ class ReviewSession:
|
|||||||
self.warnings.append(message)
|
self.warnings.append(message)
|
||||||
self.console.print(f"[yellow]{message}[/yellow]")
|
self.console.print(f"[yellow]{message}[/yellow]")
|
||||||
|
|
||||||
|
def _adopt(self, row: dict) -> bool:
|
||||||
|
"""Swap `row` (possibly an orphaned reference from before a reload)
|
||||||
|
back into self.rows by identity or key. Key collisions (duplicate
|
||||||
|
two-edition rows) prefer a still-undecided slot. False = row is gone
|
||||||
|
from the file entirely."""
|
||||||
|
if any(r is row for r in self.rows):
|
||||||
|
return True
|
||||||
|
key = (row["title_raw"], row["source_photos"])
|
||||||
|
candidates = [
|
||||||
|
i
|
||||||
|
for i, r in enumerate(self.rows)
|
||||||
|
if (r["title_raw"], r["source_photos"]) == key
|
||||||
|
]
|
||||||
|
if not candidates:
|
||||||
|
return False
|
||||||
|
undecided = [
|
||||||
|
i
|
||||||
|
for i in candidates
|
||||||
|
if self.rows[i]["match_status"] in ("ambiguous", "unmatched", "merged")
|
||||||
|
]
|
||||||
|
self.rows[(undecided or candidates)[0]] = row
|
||||||
|
return True
|
||||||
|
|
||||||
def _save(self, row: dict | None = None) -> None:
|
def _save(self, row: dict | None = None) -> None:
|
||||||
"""Atomic write of the in-memory rows. If another process rewrote the
|
"""Atomic write of the in-memory rows. A TUI loop iterates row
|
||||||
file since we loaded (resolve in a second terminal), reload first and
|
references snapshotted before any reload, and reload_if_changed can
|
||||||
re-apply `row` — the one decision being saved — by identity, so
|
swap self.rows at every save — so the decided row must always be
|
||||||
neither side's work is silently lost."""
|
re-adopted into the CURRENT list, or the decision would be counted
|
||||||
if self.reload_if_changed() and row is not None:
|
but never written."""
|
||||||
key = (row["title_raw"], row["source_photos"])
|
self.reload_if_changed()
|
||||||
for i, fresh in enumerate(self.rows):
|
if row is not None and not self._adopt(row):
|
||||||
if (fresh["title_raw"], fresh["source_photos"]) == key:
|
self._warn(
|
||||||
self.rows[i] = row
|
f"{row['title_raw']!r} disappeared from matches.csv while "
|
||||||
break
|
"you decided — decision NOT saved"
|
||||||
else:
|
)
|
||||||
self._warn(
|
return
|
||||||
f"{row['title_raw']!r} disappeared from matches.csv while "
|
|
||||||
"you decided — decision NOT saved"
|
|
||||||
)
|
|
||||||
return
|
|
||||||
try:
|
try:
|
||||||
write_matches(self.cfg.matches_path, self.rows)
|
write_matches(self.cfg.matches_path, self.rows)
|
||||||
except OSError:
|
except OSError:
|
||||||
|
|||||||
@@ -282,12 +282,17 @@ async function post(url, body) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!res.ok) {
|
if (!res.ok) {
|
||||||
const detail = await res.json().then(d => d.detail).catch(() => res.statusText);
|
const detail = await res.json().then(d => d.detail).catch(() => null);
|
||||||
alert("That didn't save: " + detail);
|
alert("That didn't save: " + (detail ?? res.statusText));
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
STATE = await res.json();
|
try {
|
||||||
render();
|
STATE = await res.json();
|
||||||
|
render();
|
||||||
|
} catch (err) {
|
||||||
|
// the decision saved server-side; only the re-render failed
|
||||||
|
errorBanner(`saved, but the page failed to refresh (${err.message || err}) — reload the page`);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function cueChips(cues) {
|
function cueChips(cues) {
|
||||||
@@ -432,6 +437,7 @@ function render() {
|
|||||||
html += `<h2>Merges <span class="count">— duplicate reads folded into one game; veto if wrong</span></h2>`;
|
html += `<h2>Merges <span class="count">— duplicate reads folded into one game; veto if wrong</span></h2>`;
|
||||||
html += s.merges.map(mg => `
|
html += s.merges.map(mg => `
|
||||||
<section class="card merge actionable" data-kind="merge"
|
<section class="card merge actionable" data-kind="merge"
|
||||||
|
data-rowix="${mg.row_ix}"
|
||||||
data-title="${esc(mg.title_raw)}" data-photos="${esc(mg.source_photos)}">
|
data-title="${esc(mg.title_raw)}" data-photos="${esc(mg.source_photos)}">
|
||||||
<div class="body">
|
<div class="body">
|
||||||
<span class="cname">${esc(mg.title_raw)}</span>
|
<span class="cname">${esc(mg.title_raw)}</span>
|
||||||
@@ -522,6 +528,7 @@ const dismiss = t => post("/api/dismiss", {
|
|||||||
});
|
});
|
||||||
const vetoMerge = card => post("/api/veto-merge", {
|
const vetoMerge = card => post("/api/veto-merge", {
|
||||||
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
||||||
|
row_ix: rowIx(card),
|
||||||
});
|
});
|
||||||
|
|
||||||
document.addEventListener("keydown", e => {
|
document.addEventListener("keydown", e => {
|
||||||
@@ -566,7 +573,7 @@ setInterval(async () => {
|
|||||||
if (++pollMisses >= 3) errorBanner(`lost contact (${err.message || err})`);
|
if (++pollMisses >= 3) errorBanner(`lost contact (${err.message || err})`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (pollMisses >= 3) showBanner(""); // recovered: clear the lost-contact banner
|
if (pollMisses >= 3) render(); // recovered: rebuild banners from state
|
||||||
pollMisses = 0;
|
pollMisses = 0;
|
||||||
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
|
if (JSON.stringify(fresh) !== JSON.stringify(STATE)) {
|
||||||
STATE = fresh;
|
STATE = fresh;
|
||||||
|
|||||||
+79
-20
@@ -21,7 +21,6 @@ domcontentloaded plus explicit element waits.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import contextlib
|
|
||||||
import csv
|
import csv
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
@@ -67,17 +66,24 @@ class UploadJob:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def key(self) -> tuple[str, str, str]:
|
def key(self) -> tuple[str, str, str]:
|
||||||
# A second copy of the same game (different version) is a distinct
|
return _key(self.action, self.bgg_id, self.collid, self.version_id)
|
||||||
# add; updates are keyed by the physical copy they amend.
|
|
||||||
if self.action == "update":
|
|
||||||
return ("update", self.collid, "")
|
def _key(
|
||||||
return ("add", self.bgg_id, self.version_id)
|
action: str, bgg_id: str, collid: str, version_id: str
|
||||||
|
) -> tuple[str, str, str]:
|
||||||
|
# A second copy of the same game (different version) is a distinct add;
|
||||||
|
# updates are keyed by the physical copy they amend. NOTE: two copies
|
||||||
|
# with the SAME (bgg_id, version) — vetoed duplicates — share a key, so
|
||||||
|
# build_queue counts completions per key instead of treating the key as
|
||||||
|
# unique.
|
||||||
|
if action == "update":
|
||||||
|
return ("update", collid, "")
|
||||||
|
return ("add", bgg_id, version_id)
|
||||||
|
|
||||||
|
|
||||||
def _job_key(row: dict) -> tuple[str, str, str]:
|
def _job_key(row: dict) -> tuple[str, str, str]:
|
||||||
if row["action"] == "update":
|
return _key(row["action"], row["bgg_id"], row["collid"], row["version_id"])
|
||||||
return ("update", row["collid"], "")
|
|
||||||
return ("add", row["bgg_id"], row["version_id"])
|
|
||||||
|
|
||||||
|
|
||||||
def _read_csv(path: Path) -> list[dict]:
|
def _read_csv(path: Path) -> list[dict]:
|
||||||
@@ -105,12 +111,21 @@ def build_queue(
|
|||||||
log_rows: list[dict],
|
log_rows: list[dict],
|
||||||
*,
|
*,
|
||||||
retry_failed: bool = False,
|
retry_failed: bool = False,
|
||||||
) -> tuple[list[UploadJob], int, int]:
|
) -> tuple[list[UploadJob], int, int, list[UploadJob]]:
|
||||||
"""Turn the diff outputs into pending jobs, minus work the log says is
|
"""Turn the diff outputs into pending jobs, minus work the log says is
|
||||||
done. Returns (jobs, skipped_done, skipped_failed)."""
|
done. Returns (jobs, skipped_done, skipped_failed, deferred) — deferred
|
||||||
latest: dict[tuple[str, str, str], str] = {}
|
being same-game updates held for a later run.
|
||||||
|
|
||||||
|
Completions are COUNTED per key, not looked up: two vetoed duplicate
|
||||||
|
copies share a key, and one logged success must complete exactly one
|
||||||
|
of them."""
|
||||||
|
done_count: Counter[tuple[str, str, str]] = Counter()
|
||||||
|
last_status: dict[tuple[str, str, str], str] = {}
|
||||||
for row in log_rows:
|
for row in log_rows:
|
||||||
latest[_job_key(row)] = row["status"]
|
k = _job_key(row)
|
||||||
|
if row["status"] in DONE_STATUSES:
|
||||||
|
done_count[k] += 1
|
||||||
|
last_status[k] = row["status"]
|
||||||
|
|
||||||
candidates = [
|
candidates = [
|
||||||
UploadJob(
|
UploadJob(
|
||||||
@@ -137,11 +152,21 @@ def build_queue(
|
|||||||
skipped_done = skipped_failed = 0
|
skipped_done = skipped_failed = 0
|
||||||
deferred: list[UploadJob] = []
|
deferred: list[UploadJob] = []
|
||||||
update_game_seen: set[str] = set()
|
update_game_seen: set[str] = set()
|
||||||
|
seen: Counter[tuple[str, str, str]] = Counter()
|
||||||
for job in candidates:
|
for job in candidates:
|
||||||
status = latest.get(job.key)
|
occurrence = seen[job.key]
|
||||||
if status in DONE_STATUSES:
|
seen[job.key] += 1
|
||||||
|
if not job.name:
|
||||||
|
# an empty name (manual id whose lookup failed) would make the
|
||||||
|
# name-driven selectors match ANY heading/row — refuse loudly
|
||||||
|
typer.echo(
|
||||||
|
f" refusing to queue bgg_id {job.bgg_id}: empty game name "
|
||||||
|
"— re-review this match so the name resolves"
|
||||||
|
)
|
||||||
|
skipped_failed += 1
|
||||||
|
elif occurrence < done_count[job.key]:
|
||||||
skipped_done += 1
|
skipped_done += 1
|
||||||
elif status == "failed" and not retry_failed:
|
elif last_status.get(job.key) == "failed" and not retry_failed:
|
||||||
skipped_failed += 1
|
skipped_failed += 1
|
||||||
elif job.action == "update" and job.bgg_id in update_game_seen:
|
elif job.action == "update" and job.bgg_id in update_game_seen:
|
||||||
# The row-edit flow finds rows by game name, not collid — a
|
# The row-edit flow finds rows by game name, not collid — a
|
||||||
@@ -281,8 +306,17 @@ class PlaywrightUploader:
|
|||||||
cancelled — when the name never shows up."""
|
cancelled — when the name never shows up."""
|
||||||
dialog.get_by_role("button", name="Set version/edition").click()
|
dialog.get_by_role("button", name="Set version/edition").click()
|
||||||
pattern = re.compile(re.escape(version_name), re.I)
|
pattern = re.compile(re.escape(version_name), re.I)
|
||||||
with contextlib.suppress(self._timeout_error): # empty list is legal
|
try:
|
||||||
dialog.get_by_role("listitem").first.wait_for(timeout=10_000)
|
dialog.get_by_role("listitem").first.wait_for(timeout=15_000)
|
||||||
|
except self._timeout_error as err:
|
||||||
|
# A version resolve found on BGG cannot legitimately be missing
|
||||||
|
# from the picker — an unrendered list means a slow page or
|
||||||
|
# changed markup. Raising keeps the attempt retryable instead
|
||||||
|
# of a terminal (and false) added_no_version.
|
||||||
|
raise RuntimeError(
|
||||||
|
"version picker never rendered — site slow or markup "
|
||||||
|
"changed; attempt is retryable"
|
||||||
|
) from err
|
||||||
for _ in range(MAX_VERSION_PAGES):
|
for _ in range(MAX_VERSION_PAGES):
|
||||||
items = dialog.get_by_role("listitem").filter(has_text=pattern)
|
items = dialog.get_by_role("listitem").filter(has_text=pattern)
|
||||||
if items.count():
|
if items.count():
|
||||||
@@ -396,7 +430,8 @@ def _process(
|
|||||||
suffix = f" — {note}" if note else ""
|
suffix = f" — {note}" if note else ""
|
||||||
typer.echo(f" {job.name}: {status}{suffix}")
|
typer.echo(f" {job.name}: {status}{suffix}")
|
||||||
if status == "failed":
|
if status == "failed":
|
||||||
consecutive = (note, consecutive[1] + 1 if note == consecutive[0] else 1)
|
kind = note.split(":", 1)[0] # exception type from _scrub format
|
||||||
|
consecutive = (kind, consecutive[1] + 1 if kind == consecutive[0] else 1)
|
||||||
if consecutive[1] >= 3:
|
if consecutive[1] >= 3:
|
||||||
typer.echo(
|
typer.echo(
|
||||||
" aborting — 3 identical consecutive failures look "
|
" aborting — 3 identical consecutive failures look "
|
||||||
@@ -418,14 +453,33 @@ def verify_uploads(log_rows: list[dict], collection: list[CollectionItem]) -> li
|
|||||||
by_collid[item.coll_id] = item
|
by_collid[item.coll_id] = item
|
||||||
|
|
||||||
problems = []
|
problems = []
|
||||||
|
added_copies: Counter[int] = Counter()
|
||||||
|
seen_add_keys: set[tuple[str, str, str]] = set()
|
||||||
latest: dict[tuple[str, str, str], dict] = {}
|
latest: dict[tuple[str, str, str], dict] = {}
|
||||||
for row in log_rows:
|
for row in log_rows:
|
||||||
latest[_job_key(row)] = row
|
k = _job_key(row)
|
||||||
|
latest[k] = row
|
||||||
|
if row["action"] == "add" and row["status"] in DONE_STATUSES:
|
||||||
|
added_copies[int(row["bgg_id"])] += 1
|
||||||
for row in latest.values():
|
for row in latest.values():
|
||||||
if row["status"] in ("added", "added_no_version"):
|
if row["status"] in ("added", "added_no_version"):
|
||||||
copies = by_object.get(int(row["bgg_id"]), [])
|
copies = by_object.get(int(row["bgg_id"]), [])
|
||||||
if not copies:
|
if not copies:
|
||||||
problems.append(f"{row['name']}: logged added but not in collection")
|
problems.append(f"{row['name']}: logged added but not in collection")
|
||||||
|
elif (
|
||||||
|
_job_key(row) not in seen_add_keys
|
||||||
|
and len(copies) < added_copies[int(row["bgg_id"])]
|
||||||
|
):
|
||||||
|
# the unverified second-copy dialog may EDIT the existing
|
||||||
|
# entry instead of creating one — a count shortfall is the
|
||||||
|
# only externally visible symptom
|
||||||
|
seen_add_keys.add(_job_key(row))
|
||||||
|
problems.append(
|
||||||
|
f"{row['name']}: {added_copies[int(row['bgg_id'])]} "
|
||||||
|
f"add(s) logged but only {len(copies)} cop"
|
||||||
|
f"{'y' if len(copies) == 1 else 'ies'} in the collection"
|
||||||
|
" — a second-copy add may have edited an existing entry"
|
||||||
|
)
|
||||||
elif (
|
elif (
|
||||||
row["status"] == "added" # no_version: absence is expected
|
row["status"] == "added" # no_version: absence is expected
|
||||||
and row["version_id"]
|
and row["version_id"]
|
||||||
@@ -494,6 +548,11 @@ def run_upload(
|
|||||||
typer.echo(f"{cfg.to_add_path} not found — run `bggpipe diff` first.")
|
typer.echo(f"{cfg.to_add_path} not found — run `bggpipe diff` first.")
|
||||||
raise typer.Exit(code=1)
|
raise typer.Exit(code=1)
|
||||||
to_add = _read_csv(cfg.to_add_path)
|
to_add = _read_csv(cfg.to_add_path)
|
||||||
|
if not cfg.to_update_path.exists():
|
||||||
|
typer.echo(
|
||||||
|
f"note: {cfg.to_update_path} not found — no version updates "
|
||||||
|
"queued (re-run `bggpipe diff` if that's unexpected)"
|
||||||
|
)
|
||||||
to_update = _read_csv(cfg.to_update_path)
|
to_update = _read_csv(cfg.to_update_path)
|
||||||
log_rows = _read_csv(log_path)
|
log_rows = _read_csv(log_path)
|
||||||
|
|
||||||
|
|||||||
+32
-17
@@ -28,8 +28,11 @@ from pydantic import BaseModel
|
|||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
|
|
||||||
from bggpipe.bgg_client import BGGClient
|
from bggpipe.bgg_client import BGGClient
|
||||||
from bggpipe.config import DEFAULT_REVIEW_PORT as DEFAULT_PORT # single home
|
from bggpipe.config import DEFAULT_REVIEW_PORT, Config
|
||||||
from bggpipe.config import Config
|
from bggpipe.models import (
|
||||||
|
CONFIDENT_VERSION_STATUSES,
|
||||||
|
RECOGNIZED_MATCH_STATUSES,
|
||||||
|
)
|
||||||
from bggpipe.review import ReviewSession
|
from bggpipe.review import ReviewSession
|
||||||
|
|
||||||
|
|
||||||
@@ -51,6 +54,13 @@ def load_thumbnails(cache_dir: Path) -> dict[int, str]:
|
|||||||
return thumbnails
|
return thumbnails
|
||||||
|
|
||||||
|
|
||||||
|
def _ix_of(rows: list[dict], row: dict) -> int:
|
||||||
|
"""Index by IDENTITY: list.index compares by ==, which returns the
|
||||||
|
first of two equal duplicate rows for both — defeating the ordinal
|
||||||
|
disambiguation row_ix exists to provide."""
|
||||||
|
return next(i for i, r in enumerate(rows) if r is row)
|
||||||
|
|
||||||
|
|
||||||
def _sighting_key(photo: str, sighting: dict) -> str:
|
def _sighting_key(photo: str, sighting: dict) -> str:
|
||||||
return "|".join(
|
return "|".join(
|
||||||
[
|
[
|
||||||
@@ -129,9 +139,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
if session.reload_if_changed():
|
if session.reload_if_changed():
|
||||||
thumbnails = load_thumbnails(cfg.cache_dir)
|
thumbnails = load_thumbnails(cfg.cache_dir)
|
||||||
|
|
||||||
def find_row(
|
def find_row(title_raw: str, source_photos: str, row_ix: int | None = None) -> dict:
|
||||||
title_raw: str, source_photos: str, row_ix: int | None = None
|
|
||||||
) -> dict:
|
|
||||||
freshen()
|
freshen()
|
||||||
# ordinal first: (title_raw, source_photos) is not unique when one
|
# ordinal first: (title_raw, source_photos) is not unique when one
|
||||||
# photo holds two editions of the same game
|
# photo holds two editions of the same game
|
||||||
@@ -139,9 +147,18 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
row = session.rows[row_ix]
|
row = session.rows[row_ix]
|
||||||
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
|
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
|
||||||
return row
|
return row
|
||||||
for row in session.rows:
|
matches = [
|
||||||
if row["title_raw"] == title_raw and row["source_photos"] == source_photos:
|
row
|
||||||
return row
|
for row in session.rows
|
||||||
|
if row["title_raw"] == title_raw and row["source_photos"] == source_photos
|
||||||
|
]
|
||||||
|
undecided = [
|
||||||
|
row
|
||||||
|
for row in matches
|
||||||
|
if row["match_status"] in ("ambiguous", "unmatched", "merged")
|
||||||
|
]
|
||||||
|
if undecided or matches:
|
||||||
|
return (undecided or matches)[0]
|
||||||
raise HTTPException(404, "row not found — matches.csv changed underneath?")
|
raise HTTPException(404, "row not found — matches.csv changed underneath?")
|
||||||
|
|
||||||
def photo_names() -> set[str]:
|
def photo_names() -> set[str]:
|
||||||
@@ -156,7 +173,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
for c in candidates:
|
for c in candidates:
|
||||||
c["thumbnail"] = thumbnails.get(c.get("bgg_id"))
|
c["thumbnail"] = thumbnails.get(c.get("bgg_id"))
|
||||||
return {
|
return {
|
||||||
"row_ix": session.rows.index(row),
|
"row_ix": _ix_of(session.rows, row),
|
||||||
"title_raw": row["title_raw"],
|
"title_raw": row["title_raw"],
|
||||||
"source_photos": row["source_photos"],
|
"source_photos": row["source_photos"],
|
||||||
"match_status": row["match_status"],
|
"match_status": row["match_status"],
|
||||||
@@ -173,7 +190,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
|
|
||||||
def version_payload(row: dict) -> dict:
|
def version_payload(row: dict) -> dict:
|
||||||
return {
|
return {
|
||||||
"row_ix": session.rows.index(row),
|
"row_ix": _ix_of(session.rows, row),
|
||||||
"title_raw": row["title_raw"],
|
"title_raw": row["title_raw"],
|
||||||
"source_photos": row["source_photos"],
|
"source_photos": row["source_photos"],
|
||||||
"bgg_name": row["bgg_name"],
|
"bgg_name": row["bgg_name"],
|
||||||
@@ -208,8 +225,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
version_updates = sum(
|
version_updates = sum(
|
||||||
1
|
1
|
||||||
for r in session.rows
|
for r in session.rows
|
||||||
if r["version_status"] in ("version_auto", "version_approved")
|
if r["version_status"] in CONFIDENT_VERSION_STATUSES and r["version_id"]
|
||||||
and r["version_id"]
|
|
||||||
)
|
)
|
||||||
available = photo_names()
|
available = photo_names()
|
||||||
sightings = []
|
sightings = []
|
||||||
@@ -223,6 +239,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
)
|
)
|
||||||
merges = [
|
merges = [
|
||||||
{
|
{
|
||||||
|
"row_ix": _ix_of(session.rows, r),
|
||||||
"title_raw": r["title_raw"],
|
"title_raw": r["title_raw"],
|
||||||
"source_photos": r["source_photos"],
|
"source_photos": r["source_photos"],
|
||||||
"merged_into": r.get("merged_into", ""),
|
"merged_into": r.get("merged_into", ""),
|
||||||
@@ -240,7 +257,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
"catalog": catalog,
|
"catalog": catalog,
|
||||||
"decisions": session.decisions,
|
"decisions": session.decisions,
|
||||||
"summary": {
|
"summary": {
|
||||||
"recognized": counts.get("auto", 0) + counts.get("approved", 0),
|
"recognized": sum(counts.get(s, 0) for s in RECOGNIZED_MATCH_STATUSES),
|
||||||
"ambiguous": counts.get("ambiguous", 0),
|
"ambiguous": counts.get("ambiguous", 0),
|
||||||
"unmatched": counts.get("unmatched", 0),
|
"unmatched": counts.get("unmatched", 0),
|
||||||
"rejected": counts.get("rejected", 0),
|
"rejected": counts.get("rejected", 0),
|
||||||
@@ -315,9 +332,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
@app.post("/api/dismiss")
|
@app.post("/api/dismiss")
|
||||||
def api_dismiss(body: DismissBody) -> dict:
|
def api_dismiss(body: DismissBody) -> dict:
|
||||||
with lock:
|
with lock:
|
||||||
dismissed.add(
|
dismissed.add(_sighting_key(body.photo, body.model_dump(exclude={"photo"})))
|
||||||
_sighting_key(body.photo, body.model_dump(exclude={"photo"}))
|
|
||||||
)
|
|
||||||
return state()
|
return state()
|
||||||
|
|
||||||
@app.get("/photos/{name}")
|
@app.get("/photos/{name}")
|
||||||
@@ -355,7 +370,7 @@ def _dev_app() -> FastAPI:
|
|||||||
def run_web_review(
|
def run_web_review(
|
||||||
cfg: Config,
|
cfg: Config,
|
||||||
*,
|
*,
|
||||||
port: int = DEFAULT_PORT,
|
port: int = DEFAULT_REVIEW_PORT,
|
||||||
dev: bool = False,
|
dev: bool = False,
|
||||||
config_path: Path | None = None,
|
config_path: Path | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
|
"""BGG client tests: canned transports, fake clocks — never online."""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import random
|
import random
|
||||||
@@ -182,3 +184,13 @@ def test_collection_item_missing_collid_refuses_to_parse(tmp_path):
|
|||||||
)
|
)
|
||||||
with pytest.raises(BGGResponseError):
|
with pytest.raises(BGGResponseError):
|
||||||
parse_collection(bad_xml)
|
parse_collection(bad_xml)
|
||||||
|
|
||||||
|
|
||||||
|
def test_malformed_xml_raises_and_is_never_cached(tmp_path):
|
||||||
|
from bggpipe.models import BGGResponseError
|
||||||
|
|
||||||
|
torn = '<items total="1"><item type="boardgame" id="13"><na'
|
||||||
|
client, _, _ = make_client(tmp_path, [(200, torn)])
|
||||||
|
with pytest.raises(BGGResponseError):
|
||||||
|
client.get_xml("search", {"query": "catan", "type": "boardgame"})
|
||||||
|
assert list((tmp_path / "cache").glob("*.xml")) == []
|
||||||
|
|||||||
+68
-8
@@ -10,6 +10,10 @@ from bggpipe.diff import compute_diff, load_snapshot_collection
|
|||||||
from bggpipe.models import CollectionItem
|
from bggpipe.models import CollectionItem
|
||||||
|
|
||||||
FIXTURES = Path(__file__).parent / "fixtures"
|
FIXTURES = Path(__file__).parent / "fixtures"
|
||||||
|
SNAPSHOT_NAMES = (
|
||||||
|
"collection_snapshot_base.xml",
|
||||||
|
"collection_snapshot_expansions.xml",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _item(object_id, coll_id, name="Game", version_id=None, own=True):
|
def _item(object_id, coll_id, name="Game", version_id=None, own=True):
|
||||||
@@ -115,10 +119,10 @@ def test_owned_with_matching_version_is_just_owned():
|
|||||||
assert not result.to_update and not result.to_add
|
assert not result.to_update and not result.to_add
|
||||||
|
|
||||||
|
|
||||||
def test_confident_version_matching_no_copy_is_a_second_copy_to_add():
|
def test_version_mismatch_with_unclaimed_copy_is_report_only():
|
||||||
# Spec: a (bgg_id, version_id) pair is owned only if a collection item
|
# ONE row, ONE copy with a different version: most likely the same
|
||||||
# matches BOTH. All copies carry different versions -> this is an
|
# physical box mis-scored. Spec: report the disagreement, touch nothing,
|
||||||
# additional physical copy; existing entries are never edited.
|
# and never risk uploading a duplicate entry.
|
||||||
result = compute_diff(
|
result = compute_diff(
|
||||||
[
|
[
|
||||||
_match(
|
_match(
|
||||||
@@ -131,10 +135,47 @@ def test_confident_version_matching_no_copy_is_a_second_copy_to_add():
|
|||||||
],
|
],
|
||||||
[_item(266192, 5, version_id=465063)],
|
[_item(266192, 5, version_id=465063)],
|
||||||
)
|
)
|
||||||
assert not result.to_update # additive only: never edit a set version
|
assert not result.to_update and not result.to_add
|
||||||
assert [r["version_id"] for r in result.to_add] == ["521212"]
|
assert result.already_owned == ["Wingspan"]
|
||||||
assert "fourth printing" in result.second_copies[0]
|
assert "fourth printing" in result.disagreements[0]
|
||||||
assert result.already_owned == []
|
|
||||||
|
|
||||||
|
def test_vetoed_duplicate_of_same_version_is_a_real_second_copy():
|
||||||
|
# Two rows, same confident version, ONE owned copy with that version:
|
||||||
|
# a human vetoed the merge ("these ARE two boxes"), so the exact-version
|
||||||
|
# match must consume the copy and the second row must become an add.
|
||||||
|
rows = [
|
||||||
|
_match("Catan", "13", vstatus="version_auto", vid="123", vname="3rd ed."),
|
||||||
|
_match("Catan", "13", vstatus="version_auto", vid="123", vname="3rd ed."),
|
||||||
|
]
|
||||||
|
result = compute_diff(rows, [_item(13, 900, version_id=123)])
|
||||||
|
assert result.already_owned == ["Catan"]
|
||||||
|
assert [r["version_id"] for r in result.to_add] == ["123"]
|
||||||
|
assert len(result.second_copies) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_bare_duplicate_beyond_owned_count_is_added_versionless():
|
||||||
|
# Two vetoed version-unknown rows, one owned copy: the extra bare row
|
||||||
|
# is a version-less second copy, not silently "already owned".
|
||||||
|
rows = [_match("Catan", "13"), _match("Catan", "13")]
|
||||||
|
result = compute_diff(rows, [_item(13, 900)])
|
||||||
|
assert result.already_owned == ["Catan"]
|
||||||
|
(added,) = result.to_add
|
||||||
|
assert added["version_id"] == ""
|
||||||
|
assert len(result.second_copies) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_bare_row_does_not_steal_versionless_copy_from_confident_update():
|
||||||
|
# ordering independence: the confident row upgrades the versionless
|
||||||
|
# copy even when a bare row of the same game appears first in the file
|
||||||
|
rows = [
|
||||||
|
_match("Catan", "13"),
|
||||||
|
_match("Catan", "13", vstatus="version_auto", vid="55", vname="5th ed."),
|
||||||
|
]
|
||||||
|
result = compute_diff(rows, [_item(13, 900), _item(13, 901)])
|
||||||
|
assert [u["version_id"] for u in result.to_update] == ["55"]
|
||||||
|
assert result.to_add == []
|
||||||
|
assert result.already_owned.count("Catan") == 2
|
||||||
|
|
||||||
|
|
||||||
def test_version_unknown_owned_by_bare_id():
|
def test_version_unknown_owned_by_bare_id():
|
||||||
@@ -237,3 +278,22 @@ def test_run_diff_outputs_feed_upload_unchanged(tmp_path, monkeypatch):
|
|||||||
assert [(j.action, j.bgg_id, j.name) for j in fake.calls] == [
|
assert [(j.action, j.bgg_id, j.name) for j in fake.calls] == [
|
||||||
("add", "266192", "Wingspan")
|
("add", "266192", "Wingspan")
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_token_without_username_says_so(tmp_path, monkeypatch, capsys):
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from bggpipe.diff import run_diff
|
||||||
|
from bggpipe.resolve import write_matches
|
||||||
|
|
||||||
|
monkeypatch.setenv("BGG_API_TOKEN", "tok")
|
||||||
|
monkeypatch.delenv("BGG_USERNAME", raising=False)
|
||||||
|
cfg = Config(data_dir=tmp_path) # bgg_username defaults to ""
|
||||||
|
fixtures = Path(__file__).parent / "fixtures"
|
||||||
|
for name in SNAPSHOT_NAMES:
|
||||||
|
shutil.copy(fixtures / name, tmp_path / name)
|
||||||
|
write_matches(cfg.matches_path, [_match("Catan", "13")])
|
||||||
|
run_diff(cfg)
|
||||||
|
out = capsys.readouterr().out
|
||||||
|
assert "BGG_API_TOKEN is set but BGG_USERNAME is not" in out
|
||||||
|
assert "No BGG_API_TOKEN" not in out # the old message was a lie here
|
||||||
|
|||||||
@@ -590,3 +590,44 @@ def test_empty_normalized_title_never_matches(client):
|
|||||||
entry = TitleEntry(title_raw="风声", title_normalized="")
|
entry = TitleEntry(title_raw="风声", title_normalized="")
|
||||||
# any cached query works; candidates must be rejected regardless of name
|
# any cached query works; candidates must be rejected regardless of name
|
||||||
assert _plausible_candidates(client, entry, "Catan") == []
|
assert _plausible_candidates(client, entry, "Catan") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_blocked_same_title_entry_defers_the_whole_group(tmp_path):
|
||||||
|
# entry1 of a two-edition title is blocked (no token); entry2 must NOT
|
||||||
|
# resolve, or its row would occupy entry1's pairing slot next run
|
||||||
|
import httpx as _httpx
|
||||||
|
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
(data_dir / "titles.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"title_raw": "Catan",
|
||||||
|
"edition_hint": "3rd edition",
|
||||||
|
"source_photos": ["a.jpg"],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"title_raw": "Catan",
|
||||||
|
"edition_hint": "5th edition",
|
||||||
|
"source_photos": ["b.jpg"],
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cfg = Config(data_dir=data_dir)
|
||||||
|
blocked_client = BGGClient(
|
||||||
|
cache_dir=tmp_path / "empty_cache",
|
||||||
|
transport=_httpx.MockTransport(
|
||||||
|
lambda req: _httpx.Response(401, text="Unauthorized")
|
||||||
|
),
|
||||||
|
sleep=lambda s: None,
|
||||||
|
)
|
||||||
|
run_resolve(cfg, client=blocked_client)
|
||||||
|
assert read_matches(cfg.matches_path) == [] # both deferred, none misplaced
|
||||||
|
|
||||||
|
|
||||||
|
def test_truncation_separator_chosen_by_position():
|
||||||
|
heads = _truncation_heads("Blorvath: Quest of the Zzyzx - 2nd Edition")
|
||||||
|
assert heads[0] == "Blorvath"
|
||||||
|
assert "Blorvath: Quest" not in heads # the comment's guarantee, now true
|
||||||
|
|||||||
@@ -382,3 +382,40 @@ def test_manual_id_unknown_to_bgg_warns_instead_of_crashing(tmp_path):
|
|||||||
assert session.rows[0]["match_status"] == "approved"
|
assert session.rows[0]["match_status"] == "approved"
|
||||||
assert session.rows[0]["bgg_id"] == "999999"
|
assert session.rows[0]["bgg_id"] == "999999"
|
||||||
assert any("no game with id 999999" in w for w in session.warnings)
|
assert any("no game with id 999999" in w for w in session.warnings)
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_tui_decision_after_external_rewrite_is_saved(tmp_path):
|
||||||
|
# THE round-2 catch: the TUI iterates row references snapshotted before
|
||||||
|
# any reload; after decision 1 triggers a reload, decisions 2..N used
|
||||||
|
# to be counted but never written.
|
||||||
|
from bggpipe.resolve import read_matches, write_matches
|
||||||
|
|
||||||
|
cfg = _setup(
|
||||||
|
tmp_path,
|
||||||
|
[
|
||||||
|
_row(title_raw="Alpha", match_status="unmatched"),
|
||||||
|
_row(title_raw="Beta", match_status="unmatched"),
|
||||||
|
_row(title_raw="Gamma", match_status="unmatched"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
session = ReviewSession(
|
||||||
|
cfg,
|
||||||
|
console=quiet_console(),
|
||||||
|
input_fn=scripted(),
|
||||||
|
client=unauthorized_client(tmp_path),
|
||||||
|
)
|
||||||
|
stale_refs = list(session.pending_rows()) # what run() iterates
|
||||||
|
|
||||||
|
external = read_matches(cfg.matches_path)
|
||||||
|
external.append(_row(title_raw="Newcomer", match_status="auto", bgg_id="7"))
|
||||||
|
write_matches(cfg.matches_path, external)
|
||||||
|
|
||||||
|
for ref in stale_refs: # decision 1 reloads; 2 and 3 are orphaned refs
|
||||||
|
session.decide_reject(ref)
|
||||||
|
|
||||||
|
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
|
||||||
|
assert [saved[t]["match_status"] for t in ("Alpha", "Beta", "Gamma")] == (
|
||||||
|
["rejected"] * 3
|
||||||
|
)
|
||||||
|
assert "Newcomer" in saved # the external row survived too
|
||||||
|
assert session.decisions == 3
|
||||||
|
|||||||
@@ -383,3 +383,45 @@ def test_real_run_without_credentials_exits_before_any_browser(tmp_path, monkeyp
|
|||||||
with pytest.raises(typer.Exit):
|
with pytest.raises(typer.Exit):
|
||||||
run_upload(cfg, sleep=lambda s: None, now=NOW) # uploader=None: real path
|
run_upload(cfg, sleep=lambda s: None, now=NOW) # uploader=None: real path
|
||||||
assert not (tmp_path / "upload_log.csv").exists()
|
assert not (tmp_path / "upload_log.csv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_key_second_copy_survives_limit_and_interrupts(tmp_path):
|
||||||
|
# two vetoed duplicate copies share ("add", bgg_id, version): one logged
|
||||||
|
# success must complete exactly ONE of them, not both
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
twin = _add_row(bgg_id="13", name="Catan", version_id="123", version_name="3rd")
|
||||||
|
_seed_data(tmp_path, to_add=[dict(twin), dict(twin)])
|
||||||
|
|
||||||
|
first = FakeUploader()
|
||||||
|
run_upload(cfg, uploader=first, limit=1, sleep=lambda s: None, now=NOW)
|
||||||
|
assert len(first.calls) == 1
|
||||||
|
|
||||||
|
second = FakeUploader()
|
||||||
|
run_upload(cfg, uploader=second, sleep=lambda s: None, now=NOW)
|
||||||
|
assert len(second.calls) == 1 # the second copy, not zero, not two
|
||||||
|
|
||||||
|
third = FakeUploader()
|
||||||
|
assert run_upload(cfg, uploader=third, sleep=lambda s: None, now=NOW) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_consecutive_failure_counter_resets_on_success(tmp_path):
|
||||||
|
class FlakyPairs(FakeUploader):
|
||||||
|
def add_game(self, job):
|
||||||
|
self.calls.append(job)
|
||||||
|
if job.bgg_id in ("1", "2", "4", "5"):
|
||||||
|
raise RuntimeError("dialog never appeared")
|
||||||
|
return "added", ""
|
||||||
|
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 7)])
|
||||||
|
fake = FlakyPairs()
|
||||||
|
run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
|
||||||
|
assert len(fake.calls) == 6 # fail,fail,ok,fail,fail,ok — never aborts
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_game_name_is_refused_not_uploaded(tmp_path):
|
||||||
|
cfg = _cfg(tmp_path)
|
||||||
|
_seed_data(tmp_path, to_add=[_add_row(bgg_id="42", name="")])
|
||||||
|
fake = FakeUploader()
|
||||||
|
results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
|
||||||
|
assert results == [] and fake.calls == []
|
||||||
|
|||||||
@@ -403,3 +403,32 @@ def test_session_warnings_surface_in_state(tmp_path):
|
|||||||
warnings = res.json()["warnings"]
|
warnings = res.json()["warnings"]
|
||||||
assert any("couldn't look up id 42" in w for w in warnings)
|
assert any("couldn't look up id 42" in w for w in warnings)
|
||||||
assert warnings == web.get("/api/state").json()["warnings"]
|
assert warnings == web.get("/api/state").json()["warnings"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_duplicate_rows_are_individually_decidable_via_row_ix(tmp_path):
|
||||||
|
# two editions of one game in one photo: byte-identical rows. The
|
||||||
|
# ordinal must land each decision on its own row.
|
||||||
|
from bggpipe.resolve import read_matches as read_m
|
||||||
|
from bggpipe.resolve import write_matches as write_m
|
||||||
|
|
||||||
|
cfg = make_cfg(tmp_path)
|
||||||
|
dup = _row(title_raw="Twins", match_status="unmatched")
|
||||||
|
write_m(cfg.matches_path, [dict(dup), dict(dup)])
|
||||||
|
web = TestClient(create_app(cfg, client=unauthorized_client(tmp_path)))
|
||||||
|
|
||||||
|
pending = web.get("/api/state").json()["pending"]
|
||||||
|
assert [p["title_raw"] for p in pending] == ["Twins", "Twins"]
|
||||||
|
assert pending[0]["row_ix"] != pending[1]["row_ix"] # identity, not ==
|
||||||
|
|
||||||
|
second = pending[1]
|
||||||
|
web.post(
|
||||||
|
"/api/decision",
|
||||||
|
json={
|
||||||
|
"title_raw": second["title_raw"],
|
||||||
|
"source_photos": second["source_photos"],
|
||||||
|
"row_ix": second["row_ix"],
|
||||||
|
"action": "reject",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
rows = read_m(cfg.matches_path)
|
||||||
|
assert [r["match_status"] for r in rows] == ["unmatched", "rejected"]
|
||||||
|
|||||||
@@ -64,6 +64,7 @@ dependencies = [
|
|||||||
{ name = "pillow" },
|
{ name = "pillow" },
|
||||||
{ name = "pillow-heif" },
|
{ name = "pillow-heif" },
|
||||||
{ name = "playwright" },
|
{ name = "playwright" },
|
||||||
|
{ name = "pydantic" },
|
||||||
{ name = "rapidfuzz" },
|
{ name = "rapidfuzz" },
|
||||||
{ name = "rich" },
|
{ name = "rich" },
|
||||||
{ name = "typer" },
|
{ name = "typer" },
|
||||||
@@ -85,6 +86,7 @@ requires-dist = [
|
|||||||
{ name = "pillow", specifier = ">=12.3.0" },
|
{ name = "pillow", specifier = ">=12.3.0" },
|
||||||
{ name = "pillow-heif", specifier = ">=1.5.0" },
|
{ name = "pillow-heif", specifier = ">=1.5.0" },
|
||||||
{ name = "playwright", specifier = ">=1.62.0" },
|
{ name = "playwright", specifier = ">=1.62.0" },
|
||||||
|
{ name = "pydantic", specifier = ">=2.13.4" },
|
||||||
{ name = "rapidfuzz", specifier = ">=3.9" },
|
{ name = "rapidfuzz", specifier = ">=3.9" },
|
||||||
{ name = "rich", specifier = ">=15.0.0" },
|
{ name = "rich", specifier = ">=15.0.0" },
|
||||||
{ name = "typer", specifier = ">=0.12" },
|
{ name = "typer", specifier = ">=0.12" },
|
||||||
|
|||||||
Reference in New Issue
Block a user