diff --git a/data/upload_log.csv b/data/upload_log.csv index 811a678..4e0ec07 100644 --- a/data/upload_log.csv +++ b/data/upload_log.csv @@ -77,3 +77,40 @@ update,177,53429542,Advanced Civilization,24972,,failed,2026-08-06T03:07:12+00:0 update,240,53429559,Britannia,24621,,failed,2026-08-06T03:08:42+00:00,"TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for get_by_role(""row"").filter(has_text=re.compile(r""Britannia"", re.IGNORECASE)).first.get_by_role(""link"", name=re.compile(r""own"", re.IGNORECASE)).first" update,71,53429530,Civilization,24006,,failed,2026-08-06T03:09:17+00:00,"TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for get_by_role(""row"").filter(has_text=re.compile(r""Civilization"", re.IGNORECASE)).first.get_by_role(""link"", name=re.compile(r""own"", re.IGNORECASE)).first" update,177,53429542,Advanced Civilization,24972,,failed,2026-08-06T03:09:50+00:00,"TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for get_by_role(""row"").filter(has_text=re.compile(r""Advanced\ Civilization"", re.IGNORECASE)).first.get_by_role(""link"", name=re.compile(r""own"", re.IGNORECASE)).first" +update,240,53429559,Britannia,24621,,updated,2026-08-06T03:17:05+00:00, +update,71,53429530,Civilization,24006,,failed,2026-08-06T03:17:11+00:00,"Error: Page.goto: net::ERR_NETWORK_CHANGED at https://boardgamegeek.com/collection/user/***?objectid=71&own=1 Call log: - navigating to ""https://boardgamegeek.com/collection/user/***?objectid=71&own=1"", waiting until ""domcontentloaded""" +update,177,53429542,Advanced Civilization,24972,,updated,2026-08-06T03:17:17+00:00, +update,71,53429530,Civilization,24006,,updated,2026-08-06T03:17:36+00:00, +update,3090,53429504,Doctor Who: The Game of Time & Space,25782,,updated,2026-08-06T03:17:55+00:00, +update,3585,53429482,Sorcerer: The Game of Magical Conflict,211154,,updated,2026-08-06T03:18:02+00:00, +update,2524,53429367,StarForce 'Alpha Centauri': Interstellar Conflict in the 25th Century,219137,,updated,2026-08-06T03:18:06+00:00, +update,224,53429816,History of the World,20727,,updated,2026-08-06T03:18:11+00:00, +update,9209,53429636,Ticket to Ride,294188,,updated,2026-08-06T03:18:17+00:00, +update,13,53430595,Catan,347121,,updated,2026-08-06T03:18:24+00:00, +update,1219,53430485,Labyrinth,488621,,updated,2026-08-06T03:18:32+00:00, +update,137909,53429649,Bugs in the Kitchen,216474,,updated,2026-08-06T03:18:38+00:00, +update,27710,53430142,Catan Dice Game,606657,,updated,2026-08-06T03:18:44+00:00, +update,169784,53429953,Castle Panic: The Dark Titan,260454,,updated,2026-08-06T03:18:49+00:00, +update,147370,53429666,Robot Turtles,231664,,updated,2026-08-06T03:18:55+00:00, +update,43443,53430559,Castle Panic,103068,,updated,2026-08-06T03:19:02+00:00, +update,24310,53429632,The Red Dragon Inn,374258,,updated,2026-08-06T03:19:06+00:00, +update,39856,53429748,Dixit,296529,,updated,2026-08-06T03:19:11+00:00, +update,35505,53430257,Walk the Plank!,781669,,updated,2026-08-06T03:19:15+00:00, +update,164,53430020,"Before I Kill You, Mister Bond",28465,,updated,2026-08-06T03:19:20+00:00, +update,181304,148198097,Mysterium,536289,,updated,2026-08-06T03:19:27+00:00, +update,65244,53429887,Forbidden Island,31208,,updated,2026-08-06T03:19:33+00:00, +update,95386,53429908,Tempurra,290572,,updated,2026-08-06T03:19:41+00:00, +update,258,53430200,Fluxx,20840,,updated,2026-08-06T03:19:47+00:00, +update,312786,148198100,Poetry for Neanderthals,514050,,updated,2026-08-06T03:19:52+00:00, +update,188614,53430332,Simon's Cat Card Game,779792,,updated,2026-08-06T03:19:57+00:00, +update,4324,53429850,Risk: The Lord of the Rings,26859,,updated,2026-08-06T03:20:01+00:00, +update,1927,53429933,Munchkin,28144,,updated,2026-08-06T03:20:06+00:00, +update,1784,53430007,Dark Cults,110926,,updated,2026-08-06T03:20:11+00:00, +update,176,53429986,Give Me the Brain!,27673,,updated,2026-08-06T03:20:16+00:00, +update,257,53430040,Kill Doctor Lucky,520364,,updated,2026-08-06T03:20:21+00:00, +update,153999,53612880,"...and then, we held hands.",280845,,updated,2026-08-06T03:20:26+00:00, +update,166976,53612866,Consentacle,410382,,updated,2026-08-06T03:20:32+00:00, +update,231302,53430652,The Cat Game,361880,,updated,2026-08-06T03:20:36+00:00, +update,40692,53430517,Small World,294178,,updated,2026-08-06T03:20:45+00:00, +update,172503,53429718,Mage Wars Academy,265107,,updated,2026-08-06T03:20:49+00:00, +update,104710,53429642,Wiz-War (Eighth Edition),117685,,updated,2026-08-06T03:20:53+00:00, diff --git a/src/bggpipe/templates/pages/help.html b/src/bggpipe/templates/pages/help.html index 4a8fbba..cf4150c 100644 --- a/src/bggpipe/templates/pages/help.html +++ b/src/bggpipe/templates/pages/help.html @@ -29,7 +29,7 @@

Photos — drag photos in, drop them in the photos/ folder, or (on a paired phone) tap the drop zone and shoot straight from the camera; camera captures get unique shelf-… names so they can never overwrite each other. Each photo has its own page listing every title read from it and any reshoot tickets — boxes seen but not identified. Photograph those up close, drop the new shot in, and extract again. Re-uploading a photo under the same file name deliberately replaces it, and the next extract run re-reads it.

Titles — every read off your shelves, alphabetized, with its status and photos. This is the proofread checkpoint: edit, split, remove. Its badge counts shaky read lines — the model wasn't sure and nothing has verified them; filter to them, then press ✓ looks right or edit each one.

Review — the decisions only you can make: which game a title is, which edition a copy is, whether two same-game reads are really one box (merges show a veto), and whether an unmatched title is a real game BGG simply doesn't have (keep locally: it joins the Library, never uploads). Keyboard-first; see shortcuts.

-

Queue — exactly what upload will do (new entries and version upgrades) and the log of everything it has done. Nothing reaches BGG that isn't visible here first. A job that fails is skipped by later runs (so one broken game can't loop forever); when any exist, the Pipeline's upload card offers a retry N failed checkbox.

+

Queue — exactly what upload will do (new entries and version upgrades) and the log of everything it has done. Nothing reaches BGG that isn't visible here first. A job that fails is skipped by later runs (so one broken game can't loop forever); when any exist, the Pipeline's upload card offers a retry N failed checkbox. Each queued row shows what upload did with it — pending, done, failed, or retired (a review decision since the last diff withdrew it). Finished rows stay listed until the next diff rebuilds the queue; the log below them is the permanent record.

Library — your enriched collection: filter by board games or RPGs. RPG matches are identified and enriched but never uploaded — BGG collections can't hold them, so they stay local citizens.

diff --git a/src/bggpipe/templates/pages/queue.html b/src/bggpipe/templates/pages/queue.html index 10bbb39..4508644 100644 --- a/src/bggpipe/templates/pages/queue.html +++ b/src/bggpipe/templates/pages/queue.html @@ -11,24 +11,48 @@ function table(headers, rows) { ${rows.join("")}`; } +// a queue row's state comes from the upload log: the CSVs are diff-time +// snapshots and never shrink as work completes +function stateChip(r) { + if (r.stale) return `retired`; + if (r.state === "done") return `done`; + if (r.state === "failed") return `failed`; + return `pending`; +} + +function tally(rows) { + const n = s => rows.filter(r => !r.stale && r.state === s).length; + const parts = [`${n("")} pending`]; + if (n("done")) parts.push(`${n("done")} done`); + if (n("failed")) parts.push(`${n("failed")} failed`); + const retired = rows.filter(r => r.stale).length; + if (retired) parts.push(`${retired} retired by review`); + return parts.join(" · "); +} + function render(q) { let html = ""; - html += `

To add — ${q.to_add.length} new collection entr${q.to_add.length === 1 ? "y" : "ies"}

`; + html += `

To add — ${tally(q.to_add)}

`; html += q.to_add.length - ? table(["game", "version", "seen in"], q.to_add.map(r => ` + ? table(["game", "version", "seen in", ""], q.to_add.map(r => ` ${esc(r.bgg_name)} · ${esc(r.bgg_id)} ${r.version_name ? esc(r.version_name) : `no version`} - ${esc((r.source_photos ?? "").split(";").join(", "))}`)) + ${esc((r.source_photos ?? "").split(";").join(", "))} + ${stateChip(r)}`)) : `

Nothing queued — run diff from the pipeline first.

`; - html += `

Version updates — ${q.to_update.length} existing entr${q.to_update.length === 1 ? "y" : "ies"} gaining a version

`; + html += `

Version updates — ${tally(q.to_update)}

`; html += q.to_update.length - ? table(["game", "version to set", "collection id"], q.to_update.map(r => ` + ? table(["game", "version to set", "collection id", ""], q.to_update.map(r => ` ${esc(r.bgg_name)} · ${esc(r.bgg_id)} ${esc(r.version_name)} - ${esc(r.collid)}`)) + ${esc(r.collid)} + ${stateChip(r)}`)) : `

No version updates pending.

`; + if (q.to_add.concat(q.to_update).some(r => r.state === "done")) + html += `

Finished jobs stay listed until the next diff + rebuilds the queue — the log below is the permanent record.

`; html += `

Upload log — every attempt ever made (${q.log.length})

`; html += q.log.length diff --git a/src/bggpipe/upload.py b/src/bggpipe/upload.py index cbcdd08..694476d 100644 --- a/src/bggpipe/upload.py +++ b/src/bggpipe/upload.py @@ -90,6 +90,40 @@ def _job_key(row: dict) -> tuple[str, str, str]: return _key(row["action"], row["bgg_id"], row["collid"], row["version_id"]) +def annotate_queue( + queue_rows: list[dict], action: str, log_rows: list[dict] +) -> list[dict]: + """Each queue row plus the outcome of its LAST upload attempt, in a + `state` field: "" (pending), "done", or "failed". to_add.csv and + to_update.csv are diff-time snapshots — nothing removes a row once its + job succeeds — so a reader without the log sees finished work as + outstanding forever.""" + last: dict[tuple[str, str, str], str] = {} + for row in log_rows: + last[_job_key(row)] = row["status"] + out = [] + for row in queue_rows: + key = _key( + action, + row.get("bgg_id", ""), + row.get("collid", ""), + row.get("version_id", ""), + ) + status = last.get(key, "") + out.append( + { + **row, + "state": "done" + if status in DONE_STATUSES + else "failed" + if status == "failed" + else "", + "last_status": status, + } + ) + return out + + def stale_jobs(queue_rows: list[dict], match_rows: list[dict]) -> dict[str, str]: """bgg_id -> why, for queued games the CURRENT matches.csv no longer endorses. to_add.csv is a snapshot from the last diff; a review diff --git a/src/bggpipe/webreview.py b/src/bggpipe/webreview.py index e7adcb6..e2e9739 100644 --- a/src/bggpipe/webreview.py +++ b/src/bggpipe/webreview.py @@ -762,10 +762,23 @@ def create_app( with path.open(newline="") as f: return list(csv.DictReader(f)) + from bggpipe.upload import annotate_queue, stale_jobs + + log = rows(cfg.upload_log_path) + to_add = rows(cfg.to_add_path) + to_update = rows(cfg.to_update_path) + # a review decision taken after the last diff retires a queued job + stale = stale_jobs(to_add + to_update, session.rows) return { - "to_add": rows(cfg.to_add_path), - "to_update": rows(cfg.to_update_path), - "log": rows(cfg.upload_log_path), + "to_add": [ + {**r, "stale": stale.get(r.get("bgg_id", ""), "")} + for r in annotate_queue(to_add, "add", log) + ], + "to_update": [ + {**r, "stale": stale.get(r.get("bgg_id", ""), "")} + for r in annotate_queue(to_update, "update", log) + ], + "log": log, } @app.get("/api/library") diff --git a/tests/test_web_dashboard.py b/tests/test_web_dashboard.py index 841089a..f7fa67d 100644 --- a/tests/test_web_dashboard.py +++ b/tests/test_web_dashboard.py @@ -266,10 +266,31 @@ def test_queue_endpoint_serves_all_three_ledgers(tmp_path): cfg = _cfg(tmp_path) (cfg.to_add_path).write_text("bgg_id,bgg_name\n13,Catan\n") q = _app(cfg).get("/api/queue").json() - assert q["to_add"] == [{"bgg_id": "13", "bgg_name": "Catan"}] + assert q["to_add"][0]["bgg_name"] == "Catan" + assert q["to_add"][0]["state"] == "" # never attempted assert q["to_update"] == [] and q["log"] == [] +def test_queue_rows_report_what_upload_already_did(tmp_path): + """to_add/to_update are diff-time snapshots: without the log, finished + work looks outstanding forever (all 36 updates still 'pending').""" + cfg = _cfg(tmp_path) + cfg.to_update_path.write_text( + "action,bgg_id,bgg_name,collid,version_id,version_name\n" + "update,240,Britannia,53429559,24621,AH second\n" + "update,71,Civilization,53429530,24006,AH first\n" + ) + cfg.upload_log_path.write_text( + "action,bgg_id,collid,name,version_id,second_copy,status,timestamp,error\n" + "update,240,53429559,Britannia,24621,,updated,t,\n" + "update,71,53429530,Civilization,24006,,failed,t,boom\n" + ) + q = _app(cfg).get("/api/queue").json() + by_name = {r["bgg_name"]: r for r in q["to_update"]} + assert by_name["Britannia"]["state"] == "done" + assert by_name["Civilization"]["state"] == "failed" + + def test_library_serves_games_sorted_or_empty(tmp_path): cfg = _cfg(tmp_path)