The Queue page reports what upload already did

All 36 version updates landed and all 36 still read as outstanding: to_add.csv and to_update.csv are diff-time snapshots that never shrink, and the page showed them without consulting the upload log. Rows now carry their last attempt's outcome — pending / done / failed, plus "retired" for jobs a later review decision withdrew — and each section heads with a tally instead of a raw row count. A note explains that finished rows persist until the next diff rebuilds the queue, and that the log is the permanent record. On Eric's data: to_update now reads 36 done, 0 pending. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016jXZFSTZQKzAC8fqpWSz9g
2026-08-05 23:23:50 -04:00
parent 0af9ae86c5
commit e44c7e1b92
6 changed files with 140 additions and 11 deletions
@@ -77,3 +77,40 @@ update,177,53429542,Advanced Civilization,24972,,failed,2026-08-06T03:07:12+00:0
 update,240,53429559,Britannia,24621,,failed,2026-08-06T03:08:42+00:00,"TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for get_by_role(""row"").filter(has_text=re.compile(r""Britannia"", re.IGNORECASE)).first.get_by_role(""link"", name=re.compile(r""own"", re.IGNORECASE)).first"
 update,71,53429530,Civilization,24006,,failed,2026-08-06T03:09:17+00:00,"TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for get_by_role(""row"").filter(has_text=re.compile(r""Civilization"", re.IGNORECASE)).first.get_by_role(""link"", name=re.compile(r""own"", re.IGNORECASE)).first"
 update,177,53429542,Advanced Civilization,24972,,failed,2026-08-06T03:09:50+00:00,"TimeoutError: Locator.click: Timeout 30000ms exceeded. Call log: - waiting for get_by_role(""row"").filter(has_text=re.compile(r""Advanced\ Civilization"", re.IGNORECASE)).first.get_by_role(""link"", name=re.compile(r""own"", re.IGNORECASE)).first"
+update,240,53429559,Britannia,24621,,updated,2026-08-06T03:17:05+00:00,
+update,71,53429530,Civilization,24006,,failed,2026-08-06T03:17:11+00:00,"Error: Page.goto: net::ERR_NETWORK_CHANGED at https://boardgamegeek.com/collection/user/***?objectid=71&own=1 Call log: - navigating to ""https://boardgamegeek.com/collection/user/***?objectid=71&own=1"", waiting until ""domcontentloaded"""
+update,177,53429542,Advanced Civilization,24972,,updated,2026-08-06T03:17:17+00:00,
+update,71,53429530,Civilization,24006,,updated,2026-08-06T03:17:36+00:00,
+update,3090,53429504,Doctor Who: The Game of Time & Space,25782,,updated,2026-08-06T03:17:55+00:00,
+update,3585,53429482,Sorcerer: The Game of Magical Conflict,211154,,updated,2026-08-06T03:18:02+00:00,
+update,2524,53429367,StarForce 'Alpha Centauri': Interstellar Conflict in the 25th Century,219137,,updated,2026-08-06T03:18:06+00:00,
+update,224,53429816,History of the World,20727,,updated,2026-08-06T03:18:11+00:00,
+update,9209,53429636,Ticket to Ride,294188,,updated,2026-08-06T03:18:17+00:00,
+update,13,53430595,Catan,347121,,updated,2026-08-06T03:18:24+00:00,
+update,1219,53430485,Labyrinth,488621,,updated,2026-08-06T03:18:32+00:00,
+update,137909,53429649,Bugs in the Kitchen,216474,,updated,2026-08-06T03:18:38+00:00,
+update,27710,53430142,Catan Dice Game,606657,,updated,2026-08-06T03:18:44+00:00,
+update,169784,53429953,Castle Panic: The Dark Titan,260454,,updated,2026-08-06T03:18:49+00:00,
+update,147370,53429666,Robot Turtles,231664,,updated,2026-08-06T03:18:55+00:00,
+update,43443,53430559,Castle Panic,103068,,updated,2026-08-06T03:19:02+00:00,
+update,24310,53429632,The Red Dragon Inn,374258,,updated,2026-08-06T03:19:06+00:00,
+update,39856,53429748,Dixit,296529,,updated,2026-08-06T03:19:11+00:00,
+update,35505,53430257,Walk the Plank!,781669,,updated,2026-08-06T03:19:15+00:00,
+update,164,53430020,"Before I Kill You, Mister Bond",28465,,updated,2026-08-06T03:19:20+00:00,
+update,181304,148198097,Mysterium,536289,,updated,2026-08-06T03:19:27+00:00,
+update,65244,53429887,Forbidden Island,31208,,updated,2026-08-06T03:19:33+00:00,
+update,95386,53429908,Tempurra,290572,,updated,2026-08-06T03:19:41+00:00,
+update,258,53430200,Fluxx,20840,,updated,2026-08-06T03:19:47+00:00,
+update,312786,148198100,Poetry for Neanderthals,514050,,updated,2026-08-06T03:19:52+00:00,
+update,188614,53430332,Simon's Cat Card Game,779792,,updated,2026-08-06T03:19:57+00:00,
+update,4324,53429850,Risk: The Lord of the Rings,26859,,updated,2026-08-06T03:20:01+00:00,
+update,1927,53429933,Munchkin,28144,,updated,2026-08-06T03:20:06+00:00,
+update,1784,53430007,Dark Cults,110926,,updated,2026-08-06T03:20:11+00:00,
+update,176,53429986,Give Me the Brain!,27673,,updated,2026-08-06T03:20:16+00:00,
+update,257,53430040,Kill Doctor Lucky,520364,,updated,2026-08-06T03:20:21+00:00,
+update,153999,53612880,"...and then, we held hands.",280845,,updated,2026-08-06T03:20:26+00:00,
+update,166976,53612866,Consentacle,410382,,updated,2026-08-06T03:20:32+00:00,
+update,231302,53430652,The Cat Game,361880,,updated,2026-08-06T03:20:36+00:00,
+update,40692,53430517,Small World,294178,,updated,2026-08-06T03:20:45+00:00,
+update,172503,53429718,Mage Wars Academy,265107,,updated,2026-08-06T03:20:49+00:00,
+update,104710,53429642,Wiz-War (Eighth Edition),117685,,updated,2026-08-06T03:20:53+00:00,
@@ -29,7 +29,7 @@
  <p><b><a href="/photos">Photos</a></b> — drag photos in, drop them in the <code>photos/</code> folder, or (on a paired phone) tap the drop zone and shoot straight from the camera; camera captures get unique <code>shelf-…</code> names so they can never overwrite each other. Each photo has its own page listing every title read from it and any reshoot tickets — boxes seen but not identified. Photograph those up close, drop the new shot in, and extract again. Re-uploading a photo under the same <i>file name</i> deliberately replaces it, and the next extract run re-reads it.</p>
  <p><b><a href="/titles">Titles</a></b> — every read off your shelves, alphabetized, with its status and photos. This is the proofread checkpoint: <a href="#curation">edit, split, remove</a>. Its badge counts <span class="chip shaky">shaky read</span> lines — the model wasn't sure and nothing has verified them; filter to them, then press <b>✓ looks right</b> or edit each one.</p>
  <p><b><a href="/review">Review</a></b> — the decisions only you can make: which game a title is, which edition a copy is, whether two same-game reads are really one box (merges show a veto), and whether an unmatched title is a real game BGG simply doesn't have (<b>keep locally</b>: it joins the Library, never uploads). Keyboard-first; see <a href="#keys">shortcuts</a>.</p>
-  <p><b><a href="/queue">Queue</a></b> — exactly what upload will do (new entries and version upgrades) and the log of everything it has done. Nothing reaches BGG that isn't visible here first. A job that fails is skipped by later runs (so one broken game can't loop forever); when any exist, the Pipeline's upload card offers a <b>retry N failed</b> checkbox.</p>
+  <p><b><a href="/queue">Queue</a></b> — exactly what upload will do (new entries and version upgrades) and the log of everything it has done. Nothing reaches BGG that isn't visible here first. A job that fails is skipped by later runs (so one broken game can't loop forever); when any exist, the Pipeline's upload card offers a <b>retry N failed</b> checkbox. Each queued row shows what upload did with it — <span class="chip open">pending</span>, <span class="chip ok">done</span>, <span class="chip no">failed</span>, or <span class="chip no">retired</span> (a review decision since the last diff withdrew it). Finished rows stay listed until the next <b>diff</b> rebuilds the queue; the log below them is the permanent record.</p>
  <p><b><a href="/library">Library</a></b> — your enriched collection: filter by board games or RPGs. RPG matches are identified and enriched but never uploaded — BGG collections can't hold them, so they stay local citizens.</p>
 </div>

@@ -11,24 +11,48 @@ function table(headers, rows) {
    ${rows.join("")}</table></div>`;
 }

+// a queue row's state comes from the upload log: the CSVs are diff-time
+// snapshots and never shrink as work completes
+function stateChip(r) {
+  if (r.stale) return `<span class="chip no" title="${esc(r.stale)}">retired</span>`;
+  if (r.state === "done") return `<span class="chip ok">done</span>`;
+  if (r.state === "failed") return `<span class="chip no">failed</span>`;
+  return `<span class="chip open">pending</span>`;
+}
+
+function tally(rows) {
+  const n = s => rows.filter(r => !r.stale && r.state === s).length;
+  const parts = [`<b>${n("")}</b> pending`];
+  if (n("done")) parts.push(`<b>${n("done")}</b> done`);
+  if (n("failed")) parts.push(`<b>${n("failed")}</b> failed`);
+  const retired = rows.filter(r => r.stale).length;
+  if (retired) parts.push(`<b>${retired}</b> retired by review`);
+  return parts.join(" · ");
+}
+
 function render(q) {
  let html = "";

-  html += `<h2>To add <span class="count">— ${q.to_add.length} new collection entr${q.to_add.length === 1 ? "y" : "ies"}</span></h2>`;
+  html += `<h2>To add <span class="count">— ${tally(q.to_add)}</span></h2>`;
  html += q.to_add.length
-    ? table(["game", "version", "seen in"], q.to_add.map(r => `
+    ? table(["game", "version", "seen in", ""], q.to_add.map(r => `
        <tr><td class="t">${esc(r.bgg_name)} <span class="meta">· ${esc(r.bgg_id)}</span></td>
        <td>${r.version_name ? esc(r.version_name) : `<span class="meta">no version</span>`}</td>
-        <td class="meta">${esc((r.source_photos ?? "").split(";").join(", "))}</td></tr>`))
+        <td class="meta">${esc((r.source_photos ?? "").split(";").join(", "))}</td>
+        <td>${stateChip(r)}</td></tr>`))
    : `<p class="empty">Nothing queued — run <b>diff</b> from the <a href="/">pipeline</a> first.</p>`;

-  html += `<h2>Version updates <span class="count">— ${q.to_update.length} existing entr${q.to_update.length === 1 ? "y" : "ies"} gaining a version</span></h2>`;
+  html += `<h2>Version updates <span class="count">— ${tally(q.to_update)}</span></h2>`;
  html += q.to_update.length
-    ? table(["game", "version to set", "collection id"], q.to_update.map(r => `
+    ? table(["game", "version to set", "collection id", ""], q.to_update.map(r => `
        <tr><td class="t">${esc(r.bgg_name)} <span class="meta">· ${esc(r.bgg_id)}</span></td>
        <td>${esc(r.version_name)}</td>
-        <td class="meta">${esc(r.collid)}</td></tr>`))
+        <td class="meta">${esc(r.collid)}</td>
+        <td>${stateChip(r)}</td></tr>`))
    : `<p class="empty">No version updates pending.</p>`;
+  if (q.to_add.concat(q.to_update).some(r => r.state === "done"))
+    html += `<p class="empty">Finished jobs stay listed until the next <b>diff</b>
+      rebuilds the queue — the log below is the permanent record.</p>`;

  html += `<h2>Upload log <span class="count">— every attempt ever made (${q.log.length})</span></h2>`;
  html += q.log.length
@@ -90,6 +90,40 @@ def _job_key(row: dict) -> tuple[str, str, str]:
    return _key(row["action"], row["bgg_id"], row["collid"], row["version_id"])


+def annotate_queue(
+    queue_rows: list[dict], action: str, log_rows: list[dict]
+) -> list[dict]:
+    """Each queue row plus the outcome of its LAST upload attempt, in a
+    `state` field: "" (pending), "done", or "failed". to_add.csv and
+    to_update.csv are diff-time snapshots — nothing removes a row once its
+    job succeeds — so a reader without the log sees finished work as
+    outstanding forever."""
+    last: dict[tuple[str, str, str], str] = {}
+    for row in log_rows:
+        last[_job_key(row)] = row["status"]
+    out = []
+    for row in queue_rows:
+        key = _key(
+            action,
+            row.get("bgg_id", ""),
+            row.get("collid", ""),
+            row.get("version_id", ""),
+        )
+        status = last.get(key, "")
+        out.append(
+            {
+                **row,
+                "state": "done"
+                if status in DONE_STATUSES
+                else "failed"
+                if status == "failed"
+                else "",
+                "last_status": status,
+            }
+        )
+    return out
+
+
 def stale_jobs(queue_rows: list[dict], match_rows: list[dict]) -> dict[str, str]:
    """bgg_id -> why, for queued games the CURRENT matches.csv no longer
    endorses. to_add.csv is a snapshot from the last diff; a review
@@ -762,10 +762,23 @@ def create_app(
            with path.open(newline="") as f:
                return list(csv.DictReader(f))

+        from bggpipe.upload import annotate_queue, stale_jobs
+
+        log = rows(cfg.upload_log_path)
+        to_add = rows(cfg.to_add_path)
+        to_update = rows(cfg.to_update_path)
+        # a review decision taken after the last diff retires a queued job
+        stale = stale_jobs(to_add + to_update, session.rows)
        return {
-            "to_add": rows(cfg.to_add_path),
-            "to_update": rows(cfg.to_update_path),
-            "log": rows(cfg.upload_log_path),
+            "to_add": [
+                {**r, "stale": stale.get(r.get("bgg_id", ""), "")}
+                for r in annotate_queue(to_add, "add", log)
+            ],
+            "to_update": [
+                {**r, "stale": stale.get(r.get("bgg_id", ""), "")}
+                for r in annotate_queue(to_update, "update", log)
+            ],
+            "log": log,
        }

    @app.get("/api/library")
@@ -266,10 +266,31 @@ def test_queue_endpoint_serves_all_three_ledgers(tmp_path):
    cfg = _cfg(tmp_path)
    (cfg.to_add_path).write_text("bgg_id,bgg_name\n13,Catan\n")
    q = _app(cfg).get("/api/queue").json()
-    assert q["to_add"] == [{"bgg_id": "13", "bgg_name": "Catan"}]
+    assert q["to_add"][0]["bgg_name"] == "Catan"
+    assert q["to_add"][0]["state"] == ""  # never attempted
    assert q["to_update"] == [] and q["log"] == []


+def test_queue_rows_report_what_upload_already_did(tmp_path):
+    """to_add/to_update are diff-time snapshots: without the log, finished
+    work looks outstanding forever (all 36 updates still 'pending')."""
+    cfg = _cfg(tmp_path)
+    cfg.to_update_path.write_text(
+        "action,bgg_id,bgg_name,collid,version_id,version_name\n"
+        "update,240,Britannia,53429559,24621,AH second\n"
+        "update,71,Civilization,53429530,24006,AH first\n"
+    )
+    cfg.upload_log_path.write_text(
+        "action,bgg_id,collid,name,version_id,second_copy,status,timestamp,error\n"
+        "update,240,53429559,Britannia,24621,,updated,t,\n"
+        "update,71,53429530,Civilization,24006,,failed,t,boom\n"
+    )
+    q = _app(cfg).get("/api/queue").json()
+    by_name = {r["bgg_name"]: r for r in q["to_update"]}
+    assert by_name["Britannia"]["state"] == "done"
+    assert by_name["Civilization"]["state"] == "failed"
+
+
 def test_library_serves_games_sorted_or_empty(tmp_path):

    cfg = _cfg(tmp_path)