Post-resolve dedupe: duplicate reads merge, review can veto
Rows resolving to the same (bgg_id, version_id — or both version-
unknown) are the same physical game read twice unless their extraction
cues conflict (two editions stay separate). The survivor is the read
whose transcription matches the BGG name; losers are marked
match_status=merged with a new merged_into column — no row is ever
deleted, and older matches.csv files without the column still read.
Downstream: diff skips merged rows but folds their photos into the
survivor's to_add provenance; enrich and the review passes ignore them.
The web UI gains a Merges section ("Jokin Ha... merged into Joking
Hazard") with a veto (v key) that restores the row as a distinct
approved match, plus a merged catalog chip and header tally.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -60,6 +60,7 @@ Each stage reads the previous stage's artifact and writes its own. Re-running a
|
|||||||
- `unmatched` — nothing plausible found.
|
- `unmatched` — nothing plausible found.
|
||||||
- Expansions: BGG returns `boardgameexpansion` as a distinct type. Keep them — I own expansions and want them in the collection — but tag them so review can catch base-game/expansion confusion (a spine reading "Wingspan Europe" must not match base Wingspan).
|
- Expansions: BGG returns `boardgameexpansion` as a distinct type. Keep them — I own expansions and want them in the collection — but tag them so review can catch base-game/expansion confusion (a spine reading "Wingspan Europe" must not match base Wingspan).
|
||||||
- Cache all BGG responses on disk (keyed by query/ID) so re-runs don't re-hit the API.
|
- Cache all BGG responses on disk (keyed by query/ID) so re-runs don't re-hit the API.
|
||||||
|
- **Post-resolve dedupe**: rows resolving to the same (bgg_id, version_id — or both version-unknown) are the same physical game read twice (typo, partial spine) unless their extraction cues conflict (two editions). Losers get `match_status=merged` + a `merged_into` column pointing at the survivor — rows never silently disappear, the survivor keeps the combined photo provenance downstream, and review surfaces every merge with a veto that restores the row as a distinct approved match.
|
||||||
- **Version resolution**: once a game ID is settled (auto or approved), fetch `/thing?id=<id>&versions=1` and score the version list against the extraction's edition cues (publisher, year, language, edition wording). Same three-way classification: a single clear winner is `version_auto`; multiple plausible → `version_ambiguous` (goes to review); no cues at all → `version_unknown` (acceptable — BGG allows collection entries with no version set, and guessing wrong is worse than leaving it blank).
|
- **Version resolution**: once a game ID is settled (auto or approved), fetch `/thing?id=<id>&versions=1` and score the version list against the extraction's edition cues (publisher, year, language, edition wording). Same three-way classification: a single clear winner is `version_auto`; multiple plausible → `version_ambiguous` (goes to review); no cues at all → `version_unknown` (acceptable — BGG allows collection entries with no version set, and guessing wrong is worse than leaving it blank).
|
||||||
- Output: `matches.csv` with columns: `title_raw, bgg_id, bgg_name, year, type, match_status (auto|ambiguous|unmatched|approved|rejected), version_id, version_name, version_status (version_auto|version_ambiguous|version_unknown|version_approved), candidates_json, version_candidates_json, source_photos`.
|
- Output: `matches.csv` with columns: `title_raw, bgg_id, bgg_name, year, type, match_status (auto|ambiguous|unmatched|approved|rejected), version_id, version_name, version_status (version_auto|version_ambiguous|version_unknown|version_approved), candidates_json, version_candidates_json, source_photos`.
|
||||||
|
|
||||||
|
|||||||
+17
-2
@@ -54,6 +54,7 @@ class DiffResult:
|
|||||||
unseen: list[CollectionItem] = field(default_factory=list)
|
unseen: list[CollectionItem] = field(default_factory=list)
|
||||||
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
|
pending: list[str] = field(default_factory=list) # ambiguous/unmatched titles
|
||||||
rejected: int = 0
|
rejected: int = 0
|
||||||
|
merged: int = 0
|
||||||
recognized: int = 0
|
recognized: int = 0
|
||||||
|
|
||||||
|
|
||||||
@@ -86,11 +87,22 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
seen_object_ids: set[int] = set()
|
seen_object_ids: set[int] = set()
|
||||||
consumed_collids: set[int] = set()
|
consumed_collids: set[int] = set()
|
||||||
|
|
||||||
|
# photos of merged-away duplicate reads belong to their survivor
|
||||||
|
merged_photos: dict[str, set[str]] = {}
|
||||||
|
for row in rows:
|
||||||
|
if row["match_status"] == "merged" and row.get("merged_into"):
|
||||||
|
merged_photos.setdefault(row["merged_into"], set()).update(
|
||||||
|
p for p in row["source_photos"].split(";") if p
|
||||||
|
)
|
||||||
|
|
||||||
for row in rows:
|
for row in rows:
|
||||||
status = row["match_status"]
|
status = row["match_status"]
|
||||||
if status == "rejected":
|
if status == "rejected":
|
||||||
result.rejected += 1
|
result.rejected += 1
|
||||||
continue
|
continue
|
||||||
|
if status == "merged":
|
||||||
|
result.merged += 1 # represented by its survivor row
|
||||||
|
continue
|
||||||
if status not in ("auto", "approved") or not row["bgg_id"]:
|
if status not in ("auto", "approved") or not row["bgg_id"]:
|
||||||
result.pending.append(row["title_raw"])
|
result.pending.append(row["title_raw"])
|
||||||
continue
|
continue
|
||||||
@@ -105,6 +117,8 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
version_id = int(row["version_id"]) if confident else None
|
version_id = int(row["version_id"]) if confident else None
|
||||||
|
|
||||||
if not copies:
|
if not copies:
|
||||||
|
photos = {p for p in row["source_photos"].split(";") if p}
|
||||||
|
photos |= merged_photos.get(row["title_raw"], set())
|
||||||
result.to_add.append(
|
result.to_add.append(
|
||||||
{
|
{
|
||||||
"bgg_id": row["bgg_id"],
|
"bgg_id": row["bgg_id"],
|
||||||
@@ -114,7 +128,7 @@ def compute_diff(rows: list[dict], collection: list[CollectionItem]) -> DiffResu
|
|||||||
"version_id": row["version_id"] if confident else "",
|
"version_id": row["version_id"] if confident else "",
|
||||||
"version_name": row["version_name"] if confident else "",
|
"version_name": row["version_name"] if confident else "",
|
||||||
"title_raw": row["title_raw"],
|
"title_raw": row["title_raw"],
|
||||||
"source_photos": row["source_photos"],
|
"source_photos": ";".join(sorted(photos)),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
continue
|
continue
|
||||||
@@ -191,11 +205,12 @@ def run_diff(cfg: Config, *, client: BGGClient | None = None) -> DiffResult:
|
|||||||
_write_csv(cfg.data_dir / "to_add.csv", TO_ADD_COLUMNS, result.to_add)
|
_write_csv(cfg.data_dir / "to_add.csv", TO_ADD_COLUMNS, result.to_add)
|
||||||
_write_csv(cfg.data_dir / "to_update.csv", TO_UPDATE_COLUMNS, result.to_update)
|
_write_csv(cfg.data_dir / "to_update.csv", TO_UPDATE_COLUMNS, result.to_update)
|
||||||
|
|
||||||
|
merged_note = f" · {result.merged} merged duplicate(s)" if result.merged else ""
|
||||||
typer.echo(
|
typer.echo(
|
||||||
f"\n{result.recognized} recognized · {len(result.already_owned)} already "
|
f"\n{result.recognized} recognized · {len(result.already_owned)} already "
|
||||||
f"owned · {len(result.to_add)} to add · {len(result.to_update)} version "
|
f"owned · {len(result.to_add)} to add · {len(result.to_update)} version "
|
||||||
f"update(s) · {len(result.pending)} pending review · "
|
f"update(s) · {len(result.pending)} pending review · "
|
||||||
f"{result.rejected} rejected"
|
f"{result.rejected} rejected{merged_note}"
|
||||||
)
|
)
|
||||||
if result.disagreements:
|
if result.disagreements:
|
||||||
typer.echo("\nVersion disagreements (left untouched):")
|
typer.echo("\nVersion disagreements (left untouched):")
|
||||||
|
|||||||
+92
-2
@@ -20,6 +20,7 @@ from rapidfuzz import fuzz
|
|||||||
|
|
||||||
from bggpipe.bgg_client import BGGAuthError, BGGClient
|
from bggpipe.bgg_client import BGGAuthError, BGGClient
|
||||||
from bggpipe.config import Config
|
from bggpipe.config import Config
|
||||||
|
from bggpipe.extract import _cues_conflict
|
||||||
from bggpipe.models import GameVersion
|
from bggpipe.models import GameVersion
|
||||||
from bggpipe.normalize import normalize_title
|
from bggpipe.normalize import normalize_title
|
||||||
|
|
||||||
@@ -43,6 +44,7 @@ MATCH_COLUMNS = [
|
|||||||
"candidates_json",
|
"candidates_json",
|
||||||
"version_candidates_json",
|
"version_candidates_json",
|
||||||
"source_photos",
|
"source_photos",
|
||||||
|
"merged_into",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -370,11 +372,86 @@ def _row_key(title_raw: str, source_photos: str) -> tuple[str, str]:
|
|||||||
return (title_raw, source_photos)
|
return (title_raw, source_photos)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class MergeEvent:
|
||||||
|
loser_title: str
|
||||||
|
survivor_title: str
|
||||||
|
bgg_name: str
|
||||||
|
bgg_id: str
|
||||||
|
|
||||||
|
|
||||||
|
def dedupe_matches(rows: list[dict], titles: list[TitleEntry]) -> list[MergeEvent]:
|
||||||
|
"""Post-resolve dedupe: rows resolving to the same (bgg_id, version_id —
|
||||||
|
or both version-unknown) are the same physical game seen twice (a typo
|
||||||
|
read, a partial spine) UNLESS their extraction cues conflict, which
|
||||||
|
means two editions. Losers are marked match_status="merged" pointing at
|
||||||
|
the survivor via merged_into — no row is ever deleted, and the review
|
||||||
|
UI can veto the merge."""
|
||||||
|
entry_by_key = {(e.title_raw, ";".join(e.source_photos)): e for e in titles}
|
||||||
|
entry_by_title: dict[str, TitleEntry] = {}
|
||||||
|
for e in titles:
|
||||||
|
entry_by_title.setdefault(e.title_raw, e)
|
||||||
|
|
||||||
|
def cues(row: dict) -> dict:
|
||||||
|
entry = entry_by_key.get(
|
||||||
|
(row["title_raw"], row["source_photos"])
|
||||||
|
) or entry_by_title.get(row["title_raw"])
|
||||||
|
if entry is None:
|
||||||
|
return {}
|
||||||
|
return {
|
||||||
|
"publisher_hint": entry.publisher_hint,
|
||||||
|
"edition_hint": entry.edition_hint,
|
||||||
|
"language_hint": entry.language_hint,
|
||||||
|
"year_hint": entry.year_hint,
|
||||||
|
}
|
||||||
|
|
||||||
|
groups: dict[tuple[str, str], list[dict]] = {}
|
||||||
|
for row in rows:
|
||||||
|
if row["match_status"] not in ("auto", "approved") or not row["bgg_id"]:
|
||||||
|
continue
|
||||||
|
confident = (
|
||||||
|
row["version_status"] in ("version_auto", "version_approved")
|
||||||
|
and row["version_id"]
|
||||||
|
)
|
||||||
|
key = (row["bgg_id"], row["version_id"] if confident else "")
|
||||||
|
groups.setdefault(key, []).append(row)
|
||||||
|
|
||||||
|
events: list[MergeEvent] = []
|
||||||
|
for (bgg_id, _version), group in groups.items():
|
||||||
|
if len(group) < 2:
|
||||||
|
continue
|
||||||
|
# survivor: the row whose transcription best matches the BGG name
|
||||||
|
group = sorted(
|
||||||
|
group,
|
||||||
|
key=lambda r: (
|
||||||
|
normalize_title(r["title_raw"]) != normalize_title(r["bgg_name"] or "")
|
||||||
|
),
|
||||||
|
)
|
||||||
|
survivor = group[0]
|
||||||
|
for loser in group[1:]:
|
||||||
|
if _cues_conflict(cues(survivor), cues(loser)):
|
||||||
|
continue # conflicting edition cues: genuinely two copies
|
||||||
|
loser["match_status"] = "merged"
|
||||||
|
loser["merged_into"] = survivor["title_raw"]
|
||||||
|
events.append(
|
||||||
|
MergeEvent(
|
||||||
|
loser_title=loser["title_raw"],
|
||||||
|
survivor_title=survivor["title_raw"],
|
||||||
|
bgg_name=survivor["bgg_name"],
|
||||||
|
bgg_id=bgg_id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
def read_matches(path: Path) -> list[dict[str, str]]:
|
def read_matches(path: Path) -> list[dict[str, str]]:
|
||||||
if not path.exists():
|
if not path.exists():
|
||||||
return []
|
return []
|
||||||
with path.open(newline="") as f:
|
with path.open(newline="") as f:
|
||||||
return list(csv.DictReader(f))
|
rows = list(csv.DictReader(f))
|
||||||
|
for row in rows: # files written before the merged_into column existed
|
||||||
|
row.setdefault("merged_into", "")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
|
def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
|
||||||
@@ -382,7 +459,9 @@ def write_matches(path: Path, rows: list[dict[str, str]]) -> None:
|
|||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
tmp = path.with_name(path.name + ".tmp")
|
tmp = path.with_name(path.name + ".tmp")
|
||||||
with tmp.open("w", newline="") as f:
|
with tmp.open("w", newline="") as f:
|
||||||
writer = csv.DictWriter(f, fieldnames=MATCH_COLUMNS, extrasaction="ignore")
|
writer = csv.DictWriter(
|
||||||
|
f, fieldnames=MATCH_COLUMNS, extrasaction="ignore", restval=""
|
||||||
|
)
|
||||||
writer.writeheader()
|
writer.writeheader()
|
||||||
writer.writerows(rows)
|
writer.writerows(rows)
|
||||||
os.replace(tmp, path)
|
os.replace(tmp, path)
|
||||||
@@ -438,6 +517,17 @@ def run_resolve(
|
|||||||
|
|
||||||
append_rows(cfg.matches_path, new_rows)
|
append_rows(cfg.matches_path, new_rows)
|
||||||
|
|
||||||
|
all_rows = read_matches(cfg.matches_path)
|
||||||
|
merges = dedupe_matches(all_rows, entries)
|
||||||
|
if merges:
|
||||||
|
write_matches(cfg.matches_path, all_rows)
|
||||||
|
typer.echo("")
|
||||||
|
for m in merges:
|
||||||
|
typer.echo(
|
||||||
|
f" merged {m.loser_title!r} into {m.survivor_title!r} — same "
|
||||||
|
f"game ({m.bgg_name}, {m.bgg_id}); veto in review if wrong"
|
||||||
|
)
|
||||||
|
|
||||||
counts: dict[str, int] = {}
|
counts: dict[str, int] = {}
|
||||||
for row in new_rows:
|
for row in new_rows:
|
||||||
counts[row.match_status] = counts.get(row.match_status, 0) + 1
|
counts[row.match_status] = counts.get(row.match_status, 0) + 1
|
||||||
|
|||||||
+16
-1
@@ -116,7 +116,22 @@ class ReviewSession:
|
|||||||
return [r for r in self.rows if r["match_status"] in ("ambiguous", "unmatched")]
|
return [r for r in self.rows if r["match_status"] in ("ambiguous", "unmatched")]
|
||||||
|
|
||||||
def version_rows(self) -> list[dict]:
|
def version_rows(self) -> list[dict]:
|
||||||
return [r for r in self.rows if r["version_status"] == "version_ambiguous"]
|
return [
|
||||||
|
r
|
||||||
|
for r in self.rows
|
||||||
|
if r["version_status"] == "version_ambiguous"
|
||||||
|
and r["match_status"] != "merged"
|
||||||
|
]
|
||||||
|
|
||||||
|
def merged_rows(self) -> list[dict]:
|
||||||
|
return [r for r in self.rows if r["match_status"] == "merged"]
|
||||||
|
|
||||||
|
def veto_merge(self, row: dict) -> None:
|
||||||
|
"""The human says these are NOT the same physical game: restore the
|
||||||
|
row as a distinct, human-confirmed match."""
|
||||||
|
row["match_status"] = "approved"
|
||||||
|
row["merged_into"] = ""
|
||||||
|
self._save()
|
||||||
|
|
||||||
def cues_for(self, title_raw: str):
|
def cues_for(self, title_raw: str):
|
||||||
return self._titles.get(title_raw)
|
return self._titles.get(title_raw)
|
||||||
|
|||||||
@@ -196,6 +196,18 @@
|
|||||||
.chip.wait { background: #f3e7cd; color: var(--brass-deep); }
|
.chip.wait { background: #f3e7cd; color: var(--brass-deep); }
|
||||||
.chip.no { background: #f0ddd8; color: var(--reject); }
|
.chip.no { background: #f0ddd8; color: var(--reject); }
|
||||||
.chip.open { background: #e4e4ef; color: #4c4c78; }
|
.chip.open { background: #e4e4ef; color: #4c4c78; }
|
||||||
|
.chip.merged { background: #e2e8e4; color: var(--felt-deep); }
|
||||||
|
|
||||||
|
/* merge notices: slim, undoable */
|
||||||
|
.card.merge { padding: .55rem 1rem; align-items: center; }
|
||||||
|
.card.merge .body { display: flex; align-items: center; gap: .8rem; flex-wrap: wrap; }
|
||||||
|
.card.merge .arrow { color: var(--ink-soft); }
|
||||||
|
.card.merge button {
|
||||||
|
font: inherit; font-size: .8rem; margin-left: auto;
|
||||||
|
background: none; border: 1px solid var(--paper-edge);
|
||||||
|
border-radius: 6px; padding: .2rem .6rem; cursor: pointer;
|
||||||
|
}
|
||||||
|
.card.merge button:hover { border-color: var(--reject); color: var(--reject); }
|
||||||
@media (max-width: 700px) {
|
@media (max-width: 700px) {
|
||||||
.card, .ticket { flex-direction: column; }
|
.card, .ticket { flex-direction: column; }
|
||||||
.shots { flex-basis: auto; }
|
.shots { flex-basis: auto; }
|
||||||
@@ -209,7 +221,7 @@
|
|||||||
<span class="keyhelp">
|
<span class="keyhelp">
|
||||||
<kbd>j</kbd>/<kbd>k</kbd> move · <kbd>1</kbd>–<kbd>9</kbd> pick ·
|
<kbd>j</kbd>/<kbd>k</kbd> move · <kbd>1</kbd>–<kbd>9</kbd> pick ·
|
||||||
<kbd>r</kbd> reject · <kbd>m</kbd> manual id · <kbd>u</kbd> unknown ·
|
<kbd>r</kbd> reject · <kbd>m</kbd> manual id · <kbd>u</kbd> unknown ·
|
||||||
<kbd>d</kbd> dismiss
|
<kbd>v</kbd> veto merge · <kbd>d</kbd> dismiss
|
||||||
</span>
|
</span>
|
||||||
</header>
|
</header>
|
||||||
<main id="main"></main>
|
<main id="main"></main>
|
||||||
@@ -341,6 +353,7 @@ function render() {
|
|||||||
`<span><b>${s.pending.length}</b> matches</span>
|
`<span><b>${s.pending.length}</b> matches</span>
|
||||||
<span><b>${s.versions.length}</b> editions</span>
|
<span><b>${s.versions.length}</b> editions</span>
|
||||||
<span><b>${s.unidentified.length}</b> reshoot</span>
|
<span><b>${s.unidentified.length}</b> reshoot</span>
|
||||||
|
${s.merges.length ? `<span><b>${s.merges.length}</b> merged</span>` : ""}
|
||||||
${s.summary.unresolved ? `<span><b>${s.summary.unresolved}</b> awaiting resolve</span>` : ""}
|
${s.summary.unresolved ? `<span><b>${s.summary.unresolved}</b> awaiting resolve</span>` : ""}
|
||||||
<span>${s.decisions} decided this sitting</span>`;
|
<span>${s.decisions} decided this sitting</span>`;
|
||||||
|
|
||||||
@@ -374,6 +387,20 @@ function render() {
|
|||||||
html += `<h2>Editions <span class="count">— optional pass, never blocks uploads</span></h2>`;
|
html += `<h2>Editions <span class="count">— optional pass, never blocks uploads</span></h2>`;
|
||||||
html += s.versions.map(versionCard).join("");
|
html += s.versions.map(versionCard).join("");
|
||||||
}
|
}
|
||||||
|
if (s.merges.length) {
|
||||||
|
html += `<h2>Merges <span class="count">— duplicate reads folded into one game; veto if wrong</span></h2>`;
|
||||||
|
html += s.merges.map(mg => `
|
||||||
|
<section class="card merge actionable" data-kind="merge"
|
||||||
|
data-title="${esc(mg.title_raw)}" data-photos="${esc(mg.source_photos)}">
|
||||||
|
<div class="body">
|
||||||
|
<span class="cname">${esc(mg.title_raw)}</span>
|
||||||
|
<span class="arrow">merged into</span>
|
||||||
|
<span class="cname">${esc(mg.merged_into)}</span>
|
||||||
|
<span class="cmeta">${esc(mg.bgg_name)} · ${esc(mg.bgg_id)}</span>
|
||||||
|
<button class="veto" title="press v"><kbd>v</kbd> veto — these are different games</button>
|
||||||
|
</div>
|
||||||
|
</section>`).join("");
|
||||||
|
}
|
||||||
if (s.unidentified.length) {
|
if (s.unidentified.length) {
|
||||||
html += `<h2>Reshoot <span class="count">— boxes seen but not identified</span></h2>`;
|
html += `<h2>Reshoot <span class="count">— boxes seen but not identified</span></h2>`;
|
||||||
html += s.unidentified.map(ticket).join("");
|
html += s.unidentified.map(ticket).join("");
|
||||||
@@ -383,6 +410,7 @@ function render() {
|
|||||||
if (c.status === "awaiting_resolve") return `<span class="chip wait">awaiting BGG</span>`;
|
if (c.status === "awaiting_resolve") return `<span class="chip wait">awaiting BGG</span>`;
|
||||||
if (c.status === "auto" || c.status === "approved") return `<span class="chip ok">${c.status}</span>`;
|
if (c.status === "auto" || c.status === "approved") return `<span class="chip ok">${c.status}</span>`;
|
||||||
if (c.status === "rejected") return `<span class="chip no">rejected</span>`;
|
if (c.status === "rejected") return `<span class="chip no">rejected</span>`;
|
||||||
|
if (c.status === "merged") return `<span class="chip merged" title="merged into ${esc(c.merged_into)}">merged → ${esc(c.merged_into)}</span>`;
|
||||||
return `<span class="chip open">${esc(c.status)}</span>`;
|
return `<span class="chip open">${esc(c.status)}</span>`;
|
||||||
};
|
};
|
||||||
html += `<h2>Catalog <span class="count">— every title extracted so far (${s.catalog.length})</span></h2>
|
html += `<h2>Catalog <span class="count">— every title extracted so far (${s.catalog.length})</span></h2>
|
||||||
@@ -417,6 +445,8 @@ function render() {
|
|||||||
version(b.closest(".card"), "unknown"));
|
version(b.closest(".card"), "unknown"));
|
||||||
m.querySelectorAll(".ticket button").forEach(b => b.onclick = () =>
|
m.querySelectorAll(".ticket button").forEach(b => b.onclick = () =>
|
||||||
dismiss(b.closest(".ticket")));
|
dismiss(b.closest(".ticket")));
|
||||||
|
m.querySelectorAll(".veto").forEach(b => b.onclick = () =>
|
||||||
|
vetoMerge(b.closest(".card")));
|
||||||
m.querySelectorAll(".rowactions input").forEach(inp => {
|
m.querySelectorAll(".rowactions input").forEach(inp => {
|
||||||
inp.onkeydown = e => {
|
inp.onkeydown = e => {
|
||||||
if (e.key === "Enter" && inp.value.trim().match(/^\d+$/)) {
|
if (e.key === "Enter" && inp.value.trim().match(/^\d+$/)) {
|
||||||
@@ -448,6 +478,9 @@ const dismiss = t => post("/api/dismiss", {
|
|||||||
photo: t.dataset.photo, location: t.dataset.location,
|
photo: t.dataset.photo, location: t.dataset.location,
|
||||||
partial_text: t.dataset.partial, art_notes: t.dataset.art,
|
partial_text: t.dataset.partial, art_notes: t.dataset.art,
|
||||||
});
|
});
|
||||||
|
const vetoMerge = card => post("/api/veto-merge", {
|
||||||
|
title_raw: card.dataset.title, source_photos: card.dataset.photos,
|
||||||
|
});
|
||||||
|
|
||||||
document.addEventListener("keydown", e => {
|
document.addEventListener("keydown", e => {
|
||||||
if (e.target.tagName === "INPUT") return;
|
if (e.target.tagName === "INPUT") return;
|
||||||
@@ -468,6 +501,7 @@ document.addEventListener("keydown", e => {
|
|||||||
else if (e.key === "r" && kind === "match") decide(card, "reject");
|
else if (e.key === "r" && kind === "match") decide(card, "reject");
|
||||||
else if (e.key === "u" && kind === "version") version(card, "unknown");
|
else if (e.key === "u" && kind === "version") version(card, "unknown");
|
||||||
else if (e.key === "d" && kind === "ticket") dismiss(card);
|
else if (e.key === "d" && kind === "ticket") dismiss(card);
|
||||||
|
else if (e.key === "v" && kind === "merge") vetoMerge(card);
|
||||||
else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); }
|
else if (e.key === "m" && kind === "match") { card.querySelector("input")?.focus(); e.preventDefault(); }
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -90,6 +90,11 @@ class VersionBody(BaseModel):
|
|||||||
version_id: int | None = None
|
version_id: int | None = None
|
||||||
|
|
||||||
|
|
||||||
|
class VetoBody(BaseModel):
|
||||||
|
title_raw: str
|
||||||
|
source_photos: str
|
||||||
|
|
||||||
|
|
||||||
class DismissBody(BaseModel):
|
class DismissBody(BaseModel):
|
||||||
photo: str
|
photo: str
|
||||||
location: str = ""
|
location: str = ""
|
||||||
@@ -168,6 +173,7 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
"bgg_id": row["bgg_id"] if row else "",
|
"bgg_id": row["bgg_id"] if row else "",
|
||||||
"bgg_name": row["bgg_name"] if row else "",
|
"bgg_name": row["bgg_name"] if row else "",
|
||||||
"version_name": row["version_name"] if row else "",
|
"version_name": row["version_name"] if row else "",
|
||||||
|
"merged_into": row.get("merged_into", "") if row else "",
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
unresolved_count = sum(
|
unresolved_count = sum(
|
||||||
@@ -189,9 +195,20 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
sightings.append(
|
sightings.append(
|
||||||
{**s, "photo": photo, "photo_exists": photo in available}
|
{**s, "photo": photo, "photo_exists": photo in available}
|
||||||
)
|
)
|
||||||
|
merges = [
|
||||||
|
{
|
||||||
|
"title_raw": r["title_raw"],
|
||||||
|
"source_photos": r["source_photos"],
|
||||||
|
"merged_into": r.get("merged_into", ""),
|
||||||
|
"bgg_name": r["bgg_name"],
|
||||||
|
"bgg_id": r["bgg_id"],
|
||||||
|
}
|
||||||
|
for r in session.merged_rows()
|
||||||
|
]
|
||||||
return {
|
return {
|
||||||
"pending": [row_payload(r) for r in session.pending_rows()],
|
"pending": [row_payload(r) for r in session.pending_rows()],
|
||||||
"versions": [version_payload(r) for r in session.version_rows()],
|
"versions": [version_payload(r) for r in session.version_rows()],
|
||||||
|
"merges": merges,
|
||||||
"unidentified": sightings,
|
"unidentified": sightings,
|
||||||
"catalog": catalog,
|
"catalog": catalog,
|
||||||
"decisions": session.decisions,
|
"decisions": session.decisions,
|
||||||
@@ -250,6 +267,14 @@ def create_app(cfg: Config, *, client: BGGClient | None = None) -> FastAPI:
|
|||||||
raise HTTPException(400, f"unknown action {body.action!r}")
|
raise HTTPException(400, f"unknown action {body.action!r}")
|
||||||
return state()
|
return state()
|
||||||
|
|
||||||
|
@app.post("/api/veto-merge")
|
||||||
|
def api_veto_merge(body: VetoBody) -> dict:
|
||||||
|
row = find_row(body.title_raw, body.source_photos)
|
||||||
|
if row["match_status"] != "merged":
|
||||||
|
raise HTTPException(400, "row is not merged")
|
||||||
|
session.veto_merge(row)
|
||||||
|
return state()
|
||||||
|
|
||||||
@app.post("/api/dismiss")
|
@app.post("/api/dismiss")
|
||||||
def api_dismiss(body: DismissBody) -> dict:
|
def api_dismiss(body: DismissBody) -> dict:
|
||||||
dismissed.add(_sighting_key(body.photo, body.model_dump(exclude={"photo"})))
|
dismissed.add(_sighting_key(body.photo, body.model_dump(exclude={"photo"})))
|
||||||
|
|||||||
@@ -164,3 +164,17 @@ def test_pending_rejected_and_unseen_are_reported():
|
|||||||
assert result.rejected == 1
|
assert result.rejected == 1
|
||||||
assert [c.object_id for c in result.unseen] == [9209] # informational
|
assert [c.object_id for c in result.unseen] == [9209] # informational
|
||||||
assert result.recognized == 1
|
assert result.recognized == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_merged_rows_are_skipped_but_photos_carry_to_survivor():
|
||||||
|
matches = [
|
||||||
|
_match("Joking Hazard", "193621"),
|
||||||
|
{**_match("Jokin Ha...", "193621", status="merged"),
|
||||||
|
"merged_into": "Joking Hazard", "source_photos": "other.jpg"},
|
||||||
|
]
|
||||||
|
result = compute_diff(matches, []) # empty collection -> to_add
|
||||||
|
assert result.merged == 1
|
||||||
|
assert result.pending == [] # merged is not "needs review"
|
||||||
|
(row,) = result.to_add
|
||||||
|
assert row["title_raw"] == "Joking Hazard"
|
||||||
|
assert row["source_photos"] == "other.jpg;x.jpg" # combined
|
||||||
|
|||||||
@@ -343,3 +343,142 @@ def test_run_resolve_saves_progress_when_token_missing(tmp_path):
|
|||||||
)
|
)
|
||||||
rows2 = run_resolve(cfg, client=full)
|
rows2 = run_resolve(cfg, client=full)
|
||||||
assert [r.title_raw for r in rows2] == ["Wingspan"]
|
assert [r.title_raw for r in rows2] == ["Wingspan"]
|
||||||
|
|
||||||
|
|
||||||
|
# -- post-resolve dedupe ------------------------------------------------
|
||||||
|
|
||||||
|
from bggpipe.bgg_client import cache_key as _cache_key # noqa: E402
|
||||||
|
from bggpipe.resolve import dedupe_matches # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _mrow(
|
||||||
|
title, bgg_id, photos, name="Joking Hazard",
|
||||||
|
vstatus="version_unknown", vid="", status="auto",
|
||||||
|
):
|
||||||
|
return {
|
||||||
|
"title_raw": title,
|
||||||
|
"bgg_id": bgg_id,
|
||||||
|
"bgg_name": name,
|
||||||
|
"year": "2016",
|
||||||
|
"type": "boardgame",
|
||||||
|
"match_status": status,
|
||||||
|
"version_id": vid,
|
||||||
|
"version_name": "",
|
||||||
|
"version_status": vstatus,
|
||||||
|
"candidates_json": "[]",
|
||||||
|
"version_candidates_json": "[]",
|
||||||
|
"source_photos": photos,
|
||||||
|
"merged_into": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _tentry(title, photos, **cues):
|
||||||
|
return TitleEntry(
|
||||||
|
title_raw=title,
|
||||||
|
title_normalized=normalize_title(title),
|
||||||
|
publisher_hint=cues.get("publisher_hint", ""),
|
||||||
|
edition_hint=cues.get("edition_hint", ""),
|
||||||
|
year_hint=cues.get("year_hint"),
|
||||||
|
language_hint=cues.get("language_hint", ""),
|
||||||
|
source_photos=tuple(photos),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_merges_typo_read_into_canonical():
|
||||||
|
rows = [
|
||||||
|
_mrow("Jokin Ha...", "193621", "a.jpg"),
|
||||||
|
_mrow("Joking Hazard", "193621", "b.jpg;c.jpg"),
|
||||||
|
_mrow("Catan", "13", "d.jpg", name="CATAN"),
|
||||||
|
]
|
||||||
|
events = dedupe_matches(rows, [])
|
||||||
|
(event,) = events
|
||||||
|
assert event.loser_title == "Jokin Ha..."
|
||||||
|
assert event.survivor_title == "Joking Hazard" # name-matching read survives
|
||||||
|
by_title = {r["title_raw"]: r for r in rows}
|
||||||
|
assert by_title["Jokin Ha..."]["match_status"] == "merged"
|
||||||
|
assert by_title["Jokin Ha..."]["merged_into"] == "Joking Hazard"
|
||||||
|
assert by_title["Joking Hazard"]["match_status"] == "auto" # untouched
|
||||||
|
assert by_title["Catan"]["match_status"] == "auto"
|
||||||
|
assert len(rows) == 3 # nothing disappears
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_respects_conflicting_edition_cues():
|
||||||
|
rows = [
|
||||||
|
_mrow("Cosmic Encounter", "40529", "a.jpg", name="Cosmic Encounter"),
|
||||||
|
_mrow("COSMIC ENCOUNTER", "40529", "b.jpg", name="Cosmic Encounter"),
|
||||||
|
]
|
||||||
|
titles = [
|
||||||
|
_tentry("Cosmic Encounter", ["a.jpg"], edition_hint="42nd Anniversary Edition"),
|
||||||
|
_tentry("COSMIC ENCOUNTER", ["b.jpg"], edition_hint="Eon 1977 edition"),
|
||||||
|
]
|
||||||
|
assert dedupe_matches(rows, titles) == []
|
||||||
|
assert all(r["match_status"] == "auto" for r in rows)
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_versions_must_agree():
|
||||||
|
# different confident versions: two physical editions, never merged
|
||||||
|
rows = [
|
||||||
|
_mrow("Wingspan", "266192", "a.jpg", vstatus="version_auto", vid="465063"),
|
||||||
|
_mrow("WINGSPAN", "266192", "b.jpg", vstatus="version_auto", vid="521212"),
|
||||||
|
]
|
||||||
|
assert dedupe_matches(rows, []) == []
|
||||||
|
# same confident version: same box seen twice
|
||||||
|
rows2 = [
|
||||||
|
_mrow("Wingspan", "266192", "a.jpg", vstatus="version_auto", vid="465063"),
|
||||||
|
_mrow("WINGSPAN", "266192", "b.jpg", vstatus="version_auto", vid="465063"),
|
||||||
|
]
|
||||||
|
assert len(dedupe_matches(rows2, [])) == 1
|
||||||
|
# confident version vs unknown: conservative, no merge
|
||||||
|
rows3 = [
|
||||||
|
_mrow("Wingspan", "266192", "a.jpg", vstatus="version_auto", vid="465063"),
|
||||||
|
_mrow("WINGSPAN", "266192", "b.jpg"),
|
||||||
|
]
|
||||||
|
assert dedupe_matches(rows3, []) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_dedupe_is_idempotent_and_skips_merged():
|
||||||
|
rows = [
|
||||||
|
_mrow("Jokin Ha...", "193621", "a.jpg"),
|
||||||
|
_mrow("Joking Hazard", "193621", "b.jpg"),
|
||||||
|
]
|
||||||
|
assert len(dedupe_matches(rows, [])) == 1
|
||||||
|
assert dedupe_matches(rows, []) == [] # second pass: nothing new
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_resolve_dedupes_and_keeps_all_rows(tmp_path):
|
||||||
|
from bggpipe.resolve import write_matches as _wm # noqa: F401
|
||||||
|
|
||||||
|
cache = tmp_path / "cache"
|
||||||
|
cache.mkdir()
|
||||||
|
wingspan_xml = (
|
||||||
|
'<items total="1"><item type="boardgame" id="266192">'
|
||||||
|
'<name type="primary" value="Wingspan"/><yearpublished value="2019"/>'
|
||||||
|
"</item></items>"
|
||||||
|
)
|
||||||
|
for query in ("Wingspan", "WINGSPAN!"):
|
||||||
|
key = _cache_key(
|
||||||
|
"search", {"query": query, "type": "boardgame,boardgameexpansion"}
|
||||||
|
)
|
||||||
|
(cache / key).write_text(wingspan_xml)
|
||||||
|
|
||||||
|
data_dir = tmp_path / "data"
|
||||||
|
data_dir.mkdir()
|
||||||
|
(data_dir / "titles.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
[
|
||||||
|
{"title_raw": "Wingspan", "source_photos": ["a.jpg"]},
|
||||||
|
{"title_raw": "WINGSPAN!", "source_photos": ["b.jpg"]}, # variant read
|
||||||
|
]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cfg = Config(data_dir=data_dir)
|
||||||
|
client = BGGClient(cache_dir=cache, transport=httpx.MockTransport(_no_network))
|
||||||
|
|
||||||
|
run_resolve(cfg, client=client)
|
||||||
|
from bggpipe.resolve import read_matches as _rm
|
||||||
|
|
||||||
|
saved = {r["title_raw"]: r for r in _rm(cfg.matches_path)}
|
||||||
|
assert len(saved) == 2 # no row disappeared
|
||||||
|
assert saved["Wingspan"]["match_status"] == "auto"
|
||||||
|
assert saved["WINGSPAN!"]["match_status"] == "merged"
|
||||||
|
assert saved["WINGSPAN!"]["merged_into"] == "Wingspan"
|
||||||
|
|||||||
@@ -286,3 +286,41 @@ def test_catalog_and_unresolved_backlog_are_visible(tmp_path):
|
|||||||
by_title = {c["title_raw"]: c for c in state["catalog"]}
|
by_title = {c["title_raw"]: c for c in state["catalog"]}
|
||||||
assert by_title["Fresh Off The Shelf"]["status"] == "awaiting_resolve"
|
assert by_title["Fresh Off The Shelf"]["status"] == "awaiting_resolve"
|
||||||
assert by_title["Citadels"]["status"] == "ambiguous"
|
assert by_title["Citadels"]["status"] == "ambiguous"
|
||||||
|
|
||||||
|
|
||||||
|
def test_merge_veto_roundtrip(tmp_path):
|
||||||
|
cfg = make_cfg(tmp_path)
|
||||||
|
rows = read_matches(cfg.matches_path)
|
||||||
|
rows.append(
|
||||||
|
_row(
|
||||||
|
title_raw="Jokin Ha...",
|
||||||
|
match_status="merged",
|
||||||
|
bgg_id="193621",
|
||||||
|
bgg_name="Joking Hazard",
|
||||||
|
source_photos="shelf.jpg",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
rows[-1]["merged_into"] = "Joking Hazard"
|
||||||
|
write_matches(cfg.matches_path, rows)
|
||||||
|
web = TestClient(create_app(cfg, client=unauthorized_client(tmp_path)))
|
||||||
|
|
||||||
|
state = web.get("/api/state").json()
|
||||||
|
(merge,) = state["merges"]
|
||||||
|
assert merge["title_raw"] == "Jokin Ha..."
|
||||||
|
assert merge["merged_into"] == "Joking Hazard"
|
||||||
|
|
||||||
|
state = web.post(
|
||||||
|
"/api/veto-merge",
|
||||||
|
json={"title_raw": "Jokin Ha...", "source_photos": "shelf.jpg"},
|
||||||
|
).json()
|
||||||
|
assert state["merges"] == []
|
||||||
|
saved = {r["title_raw"]: r for r in read_matches(cfg.matches_path)}
|
||||||
|
assert saved["Jokin Ha..."]["match_status"] == "approved"
|
||||||
|
assert saved["Jokin Ha..."]["merged_into"] == ""
|
||||||
|
|
||||||
|
# veto on a non-merged row is refused
|
||||||
|
bad = web.post(
|
||||||
|
"/api/veto-merge",
|
||||||
|
json={"title_raw": "Jokin Ha...", "source_photos": "shelf.jpg"},
|
||||||
|
)
|
||||||
|
assert bad.status_code == 400
|
||||||
|
|||||||
Reference in New Issue
Block a user