Count pending work: BGG's export lags, so raw queue rows lie

The card read "1 version updates" while upload said "skipping 1 already
done". Both were right. Recon on the live site shows the update DID
apply — the version cell reads "English first edition Year: 2012" and
its radio is checked — but BGG's XML collection export still reports
that collid with no version, even on a forced refresh. diff reads the
API, so it re-queued finished work; the log correctly refused it.

Nothing to fix in the flow: the pipeline card now counts PENDING jobs
(queue rows minus what the log completed) for both to_add and
to_update, reports outstanding failures rather than every failure ever
logged, and when everything queued is already applied it says so and
names the cause. Documented under "BGG's collection export lags the
site" so the next person doesn't chase it as a bug.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016jXZFSTZQKzAC8fqpWSz9g
This commit is contained in:
Eric Wagoner
2026-08-05 23:29:25 -04:00
co-authored by Claude Fable 5
parent e44c7e1b92
commit b5e13be335
5 changed files with 65 additions and 39 deletions
+7 -2
View File
@@ -42,8 +42,13 @@ function render() {
const uploadFacts = `<b>${P.to_add}</b> to add · <b>${P.to_update}</b> version updates
${log.added || log.added_no_version ? `· <b>${(log.added ?? 0) + (log.added_no_version ?? 0)}</b> added` : ""}
${log.failed ? `· <b>${log.failed}</b> failed` : ""}
· <a href="/queue">inspect the queue</a>`;
${P.upload_failed ? `· <b>${P.upload_failed}</b> failed` : ""}
· <a href="/queue">inspect the queue</a>
${!P.to_add && !P.to_update && P.queued_total
? `<br><span class="meta">${P.queued_total} queued row(s) already applied —
the next <b>diff</b> clears them (BGG's collection export can lag the
site by a while)</span>`
: ""}`;
document.getElementById("stages").innerHTML = [
stageCard(1, "extract", `read titles off <b>${P.photos}</b> <a href="/photos">photo(s)</a> — <b>${P.titles}</b> so far`
+18 -2
View File
@@ -786,6 +786,15 @@ def create_app(
games = read_games()
return sorted(games.values(), key=lambda g: (g.get("name") or "").casefold())
def _pending(path: Path, action: str, log_rows: list[dict]) -> int:
from bggpipe.upload import annotate_queue
if not path.exists():
return 0
with path.open(newline="") as f:
rows = list(csv.DictReader(f))
return sum(1 for r in annotate_queue(rows, action, log_rows) if not r["state"])
def _csv_count(path: Path) -> int:
if not path.exists():
return 0
@@ -799,6 +808,7 @@ def create_app(
match_counts = Counter(row["match_status"] for row in session.rows)
log_counts: Counter[str] = Counter()
failed_now = 0
log_rows: list[dict] = []
log_path = cfg.upload_log_path
if log_path.exists():
from bggpipe.upload import outstanding_failures
@@ -855,8 +865,14 @@ def create_app(
"matches": dict(match_counts),
"pending_review": len(session.pending_rows())
+ len(session.version_rows()),
"to_add": _csv_count(cfg.to_add_path),
"to_update": _csv_count(cfg.to_update_path),
# PENDING work, not raw rows: the queue CSVs are diff-time
# snapshots that keep listing finished jobs (and BGG's
# collection export lags the site, so a re-run diff can
# re-queue work that already landed)
"to_add": _pending(cfg.to_add_path, "add", log_rows),
"to_update": _pending(cfg.to_update_path, "update", log_rows),
"queued_total": _csv_count(cfg.to_add_path)
+ _csv_count(cfg.to_update_path),
"upload_log": dict(log_counts),
"upload_failed": failed_now,
"games": games,