diff --git a/data/to_update.csv b/data/to_update.csv index c98b97a..7f9d3bf 100644 --- a/data/to_update.csv +++ b/data/to_update.csv @@ -1,37 +1,2 @@ collid,bgg_id,bgg_name,version_id,version_name -53429559,240,Britannia,24621,Avalon Hill second edition -53429530,71,Civilization,24006,Avalon Hill first edition 1981 -53429542,177,Advanced Civilization,24972,English edition -53429504,3090,Doctor Who: The Game of Time & Space,25782,English edition -53429482,3585,Sorcerer: The Game of Magical Conflict,211154,SPI Designer's Edition -53429367,2524,StarForce 'Alpha Centauri': Interstellar Conflict in the 25th Century,219137,SPI boxed designer's edition -53429816,224,History of the World,20727,Avalon Hill/Hasbro edition -53429636,9209,Ticket to Ride,294188,English edition 2012 with Spiel des Jahres logo -53430595,13,Catan,347121,English edition 2016 -53430485,1219,Labyrinth,488621,English edition 2017 -53429649,137909,Bugs in the Kitchen,216474,English first edition -53430142,27710,Catan Dice Game,606657,English edition 2018 -53429953,169784,Castle Panic: The Dark Titan,260454,English first edition -53429666,147370,Robot Turtles,231664,English second edition -53430559,43443,Castle Panic,103068,"English edition, fourth printing" -53429632,24310,The Red Dragon Inn,374258,English fifth edition -53429748,39856,Dixit,296529,Asmodee/Libellud English edition 2015 -53430257,35505,Walk the Plank!,781669,English MDG-4311 edition -53430020,164,"Before I Kill You, Mister Bond",28465,"Cheapass black and white ""Better Edition"" edition" -148198097,181304,Mysterium,536289,English edition 2018-2 -53429887,65244,Forbidden Island,31208,English edition 051210 -53429908,95386,Tempurra,290572,English edition -53430200,258,Fluxx,20840,Looney Labs Version v2.1 -148198100,312786,Poetry for Neanderthals,514050,English edition 2020 -53430332,188614,Simon's Cat Card Game,779792,"English edition, second printing" -53429850,4324,Risk: The Lord of the Rings,26859,English edition 2002 -53429933,1927,Munchkin,28144,"English edition 2009, 15th printing" -53430007,1784,Dark Cults,110926,First edition -53429986,176,Give Me the Brain!,27673,Czech edition -53430040,257,Kill Doctor Lucky,520364,"Cheapass envelope edition, second printing" -53612880,153999,"...and then, we held hands.",280845,English edition -53612866,166976,Consentacle,410382,English Kickstarter edition -53430652,231302,The Cat Game,361880,English/French edition -53430517,40692,Small World,294178,English third edition -53429718,172503,Mage Wars Academy,265107,English edition 53429642,104710,Wiz-War (Eighth Edition),117685,English first edition diff --git a/docs/bgg-upload-flow.md b/docs/bgg-upload-flow.md index be866a5..799f9be 100644 --- a/docs/bgg-upload-flow.md +++ b/docs/bgg-upload-flow.md @@ -167,3 +167,22 @@ cannot say which copy it edits. The collection table can: This is strictly additive: it sets one field on one collid and cannot create a duplicate entry. + +### BGG's collection export lags the site + +After a successful update the website shows the new version immediately +(the row's version cell reads e.g. "English first edition Year: 2012", +and reopening the editor shows that radio checked), but the XML API's +`/collection` export can still report the entry with no version — even +with a cache-busting re-request. `diff` reads the API, so it will +re-queue work that has already landed. + +Consequences to keep in mind rather than "fix": + +- The upload log is the authority on what this tool did; the API is the + authority on what BGG has published. They disagree for a while. +- `build_queue` skipping a job whose log says done is CORRECT here — the + work is applied, and re-running it would be a no-op at best. +- Anything user-facing should count PENDING jobs (queue rows minus what + the log completed), never raw queue rows, or settled work reads as + outstanding until the next diff. diff --git a/src/bggpipe/templates/pages/pipeline.html b/src/bggpipe/templates/pages/pipeline.html index e9719c4..7f39f50 100644 --- a/src/bggpipe/templates/pages/pipeline.html +++ b/src/bggpipe/templates/pages/pipeline.html @@ -42,8 +42,13 @@ function render() { const uploadFacts = `${P.to_add} to add · ${P.to_update} version updates ${log.added || log.added_no_version ? `· ${(log.added ?? 0) + (log.added_no_version ?? 0)} added` : ""} - ${log.failed ? `· ${log.failed} failed` : ""} - · inspect the queue`; + ${P.upload_failed ? `· ${P.upload_failed} failed` : ""} + · inspect the queue + ${!P.to_add && !P.to_update && P.queued_total + ? `
${P.queued_total} queued row(s) already applied — + the next diff clears them (BGG's collection export can lag the + site by a while)` + : ""}`; document.getElementById("stages").innerHTML = [ stageCard(1, "extract", `read titles off ${P.photos} photo(s)${P.titles} so far` diff --git a/src/bggpipe/webreview.py b/src/bggpipe/webreview.py index e2e9739..4d9c1f0 100644 --- a/src/bggpipe/webreview.py +++ b/src/bggpipe/webreview.py @@ -786,6 +786,15 @@ def create_app( games = read_games() return sorted(games.values(), key=lambda g: (g.get("name") or "").casefold()) + def _pending(path: Path, action: str, log_rows: list[dict]) -> int: + from bggpipe.upload import annotate_queue + + if not path.exists(): + return 0 + with path.open(newline="") as f: + rows = list(csv.DictReader(f)) + return sum(1 for r in annotate_queue(rows, action, log_rows) if not r["state"]) + def _csv_count(path: Path) -> int: if not path.exists(): return 0 @@ -799,6 +808,7 @@ def create_app( match_counts = Counter(row["match_status"] for row in session.rows) log_counts: Counter[str] = Counter() failed_now = 0 + log_rows: list[dict] = [] log_path = cfg.upload_log_path if log_path.exists(): from bggpipe.upload import outstanding_failures @@ -855,8 +865,14 @@ def create_app( "matches": dict(match_counts), "pending_review": len(session.pending_rows()) + len(session.version_rows()), - "to_add": _csv_count(cfg.to_add_path), - "to_update": _csv_count(cfg.to_update_path), + # PENDING work, not raw rows: the queue CSVs are diff-time + # snapshots that keep listing finished jobs (and BGG's + # collection export lags the site, so a re-run diff can + # re-queue work that already landed) + "to_add": _pending(cfg.to_add_path, "add", log_rows), + "to_update": _pending(cfg.to_update_path, "update", log_rows), + "queued_total": _csv_count(cfg.to_add_path) + + _csv_count(cfg.to_update_path), "upload_log": dict(log_counts), "upload_failed": failed_now, "games": games, diff --git a/tests/test_web_dashboard.py b/tests/test_web_dashboard.py index f7fa67d..6cb9abb 100644 --- a/tests/test_web_dashboard.py +++ b/tests/test_web_dashboard.py @@ -649,3 +649,24 @@ def test_photo_detail_page_serves_with_photos_nav_active(tmp_path): (current,) = re.findall(r'