Count pending work: BGG's export lags, so raw queue rows lie
The card read "1 version updates" while upload said "skipping 1 already done". Both were right. Recon on the live site shows the update DID apply — the version cell reads "English first edition Year: 2012" and its radio is checked — but BGG's XML collection export still reports that collid with no version, even on a forced refresh. diff reads the API, so it re-queued finished work; the log correctly refused it. Nothing to fix in the flow: the pipeline card now counts PENDING jobs (queue rows minus what the log completed) for both to_add and to_update, reports outstanding failures rather than every failure ever logged, and when everything queued is already applied it says so and names the cause. Documented under "BGG's collection export lags the site" so the next person doesn't chase it as a bug. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016jXZFSTZQKzAC8fqpWSz9g
This commit is contained in:
co-authored by
Claude Fable 5
parent
e44c7e1b92
commit
b5e13be335
@@ -42,8 +42,13 @@ function render() {
|
||||
|
||||
const uploadFacts = `<b>${P.to_add}</b> to add · <b>${P.to_update}</b> version updates
|
||||
${log.added || log.added_no_version ? `· <b>${(log.added ?? 0) + (log.added_no_version ?? 0)}</b> added` : ""}
|
||||
${log.failed ? `· <b>${log.failed}</b> failed` : ""}
|
||||
· <a href="/queue">inspect the queue</a>`;
|
||||
${P.upload_failed ? `· <b>${P.upload_failed}</b> failed` : ""}
|
||||
· <a href="/queue">inspect the queue</a>
|
||||
${!P.to_add && !P.to_update && P.queued_total
|
||||
? `<br><span class="meta">${P.queued_total} queued row(s) already applied —
|
||||
the next <b>diff</b> clears them (BGG's collection export can lag the
|
||||
site by a while)</span>`
|
||||
: ""}`;
|
||||
|
||||
document.getElementById("stages").innerHTML = [
|
||||
stageCard(1, "extract", `read titles off <b>${P.photos}</b> <a href="/photos">photo(s)</a> — <b>${P.titles}</b> so far`
|
||||
|
||||
@@ -786,6 +786,15 @@ def create_app(
|
||||
games = read_games()
|
||||
return sorted(games.values(), key=lambda g: (g.get("name") or "").casefold())
|
||||
|
||||
def _pending(path: Path, action: str, log_rows: list[dict]) -> int:
|
||||
from bggpipe.upload import annotate_queue
|
||||
|
||||
if not path.exists():
|
||||
return 0
|
||||
with path.open(newline="") as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
return sum(1 for r in annotate_queue(rows, action, log_rows) if not r["state"])
|
||||
|
||||
def _csv_count(path: Path) -> int:
|
||||
if not path.exists():
|
||||
return 0
|
||||
@@ -799,6 +808,7 @@ def create_app(
|
||||
match_counts = Counter(row["match_status"] for row in session.rows)
|
||||
log_counts: Counter[str] = Counter()
|
||||
failed_now = 0
|
||||
log_rows: list[dict] = []
|
||||
log_path = cfg.upload_log_path
|
||||
if log_path.exists():
|
||||
from bggpipe.upload import outstanding_failures
|
||||
@@ -855,8 +865,14 @@ def create_app(
|
||||
"matches": dict(match_counts),
|
||||
"pending_review": len(session.pending_rows())
|
||||
+ len(session.version_rows()),
|
||||
"to_add": _csv_count(cfg.to_add_path),
|
||||
"to_update": _csv_count(cfg.to_update_path),
|
||||
# PENDING work, not raw rows: the queue CSVs are diff-time
|
||||
# snapshots that keep listing finished jobs (and BGG's
|
||||
# collection export lags the site, so a re-run diff can
|
||||
# re-queue work that already landed)
|
||||
"to_add": _pending(cfg.to_add_path, "add", log_rows),
|
||||
"to_update": _pending(cfg.to_update_path, "update", log_rows),
|
||||
"queued_total": _csv_count(cfg.to_add_path)
|
||||
+ _csv_count(cfg.to_update_path),
|
||||
"upload_log": dict(log_counts),
|
||||
"upload_failed": failed_now,
|
||||
"games": games,
|
||||
|
||||
Reference in New Issue
Block a user