Stage 5 upload: Playwright-driven adds and version updates

Queue from to_add/to_update minus upload_log.csv (append-per-attempt, so runs resume); per-game failure isolation with 2-4s pacing; --dry-run/--verify/--retry-failed/--limit; stub-fixture marker blocks real runs, dry-run warns. Headed browser by default: live recon showed Cloudflare Turnstile hard-blocks headless, and BGG never reaches networkidle. Login selectors verified anonymously; version-picker pagination and the collection-row update flow remain unverified until real data exists. Client collection fetches gain a refresh passthrough so --verify sees the live collection, not cache. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-01 19:03:25 -04:00
parent 196862e243
commit 3ca7e7f650
8 changed files with 968 additions and 19 deletions
@@ -0,0 +1,292 @@
+"""Upload-stage tests: queue building, idempotency via upload_log.csv, the
+stub-fixture guard, per-game failure isolation, pacing, and verify — all
+against a fake uploader. No browser, no network."""
+
+from __future__ import annotations
+
+import csv
+import random
+from pathlib import Path
+
+import pytest
+import typer
+
+from bggpipe.config import Config
+from bggpipe.models import CollectionItem
+from bggpipe.upload import (
+    UPLOAD_LOG_COLUMNS,
+    UploadJob,
+    _scrub,
+    build_queue,
+    run_upload,
+    verify_uploads,
+)
+
+NOW = lambda: "2026-08-01T00:00:00+00:00"  # noqa: E731
+
+
+def _cfg(tmp_path: Path) -> Config:
+    return Config(bgg_username="tester", data_dir=tmp_path)
+
+
+def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None:
+    with path.open("w", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=columns)
+        writer.writeheader()
+        writer.writerows(rows)
+
+
+def _add_row(bgg_id="1", name="Wingspan", version_id="", version_name=""):
+    return {
+        "bgg_id": bgg_id,
+        "bgg_name": name,
+        "year": "2019",
+        "type": "boardgame",
+        "version_id": version_id,
+        "version_name": version_name,
+        "title_raw": name.upper(),
+        "source_photos": "x.jpg",
+    }
+
+
+def _update_row(collid="9", bgg_id="2", name="Britannia", vid="25", vname="AH ed."):
+    return {
+        "collid": collid,
+        "bgg_id": bgg_id,
+        "bgg_name": name,
+        "version_id": vid,
+        "version_name": vname,
+    }
+
+
+def _log_row(action="add", bgg_id="1", collid="", version_id="", status="added"):
+    return {
+        "action": action,
+        "bgg_id": bgg_id,
+        "collid": collid,
+        "name": "Game",
+        "version_id": version_id,
+        "status": status,
+        "timestamp": NOW(),
+        "error": "",
+    }
+
+
+def _seed_data(tmp_path: Path, to_add=None, to_update=None, log=None) -> None:
+    _write_csv(
+        tmp_path / "to_add.csv",
+        list(_add_row().keys()),
+        to_add if to_add is not None else [],
+    )
+    _write_csv(
+        tmp_path / "to_update.csv",
+        list(_update_row().keys()),
+        to_update if to_update is not None else [],
+    )
+    if log is not None:
+        _write_csv(tmp_path / "upload_log.csv", UPLOAD_LOG_COLUMNS, log)
+
+
+class FakeUploader:
+    """Records jobs; raises for names listed in `failures`."""
+
+    def __init__(self, failures: set[str] | None = None):
+        self.calls: list[UploadJob] = []
+        self.failures = failures or set()
+
+    def _handle(self, job: UploadJob, status: str) -> tuple[str, str]:
+        self.calls.append(job)
+        if job.name in self.failures:
+            raise RuntimeError("dialog never appeared")
+        return status, ""
+
+    def add_game(self, job: UploadJob) -> tuple[str, str]:
+        return self._handle(job, "added")
+
+    def update_entry(self, job: UploadJob) -> tuple[str, str]:
+        return self._handle(job, "updated")
+
+
+# -- build_queue --------------------------------------------------------
+
+
+def test_build_queue_skips_logged_successes():
+    jobs, done, failed = build_queue(
+        [_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")],
+        [_update_row(collid="9")],
+        [
+            _log_row(action="add", bgg_id="1", status="added"),
+            _log_row(action="update", bgg_id="2", collid="9", status="updated"),
+        ],
+    )
+    assert [j.bgg_id for j in jobs] == ["2"]
+    assert done == 2
+    assert failed == 0
+
+
+def test_build_queue_second_copy_is_a_distinct_job():
+    # Same game, different version: a separate physical copy, so a
+    # logged add of one version must not swallow the other.
+    jobs, done, _ = build_queue(
+        [
+            _add_row(bgg_id="1", version_id="10", version_name="First ed."),
+            _add_row(bgg_id="1", version_id="11", version_name="Second ed."),
+        ],
+        [],
+        [_log_row(action="add", bgg_id="1", version_id="10", status="added")],
+    )
+    assert [j.version_id for j in jobs] == ["11"]
+    assert done == 1
+
+
+def test_build_queue_failures_need_retry_flag():
+    log = [_log_row(action="add", bgg_id="1", status="failed")]
+    jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log)
+    assert jobs == [] and skipped == 1
+    jobs, _, skipped = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
+    assert len(jobs) == 1 and skipped == 0
+
+
+def test_build_queue_latest_log_entry_wins():
+    # failed then added on retry -> done, not retriable
+    log = [
+        _log_row(action="add", bgg_id="1", status="failed"),
+        _log_row(action="add", bgg_id="1", status="added"),
+    ]
+    jobs, done, _ = build_queue([_add_row(bgg_id="1")], [], log, retry_failed=True)
+    assert jobs == [] and done == 1
+
+
+# -- stub-fixture guard -------------------------------------------------
+
+
+def test_real_run_refuses_while_stub_marker_exists(tmp_path):
+    cfg = _cfg(tmp_path)
+    cfg.cache_dir.mkdir(parents=True)
+    (cfg.cache_dir / "STUB_FIXTURES.marker").write_text("stub")
+    _seed_data(tmp_path, to_add=[_add_row()])
+    with pytest.raises(typer.Exit):
+        run_upload(cfg, uploader=FakeUploader(), sleep=lambda s: None, now=NOW)
+
+
+def test_dry_run_allowed_with_stub_marker_and_writes_nothing(tmp_path, capsys):
+    cfg = _cfg(tmp_path)
+    cfg.cache_dir.mkdir(parents=True)
+    (cfg.cache_dir / "STUB_FIXTURES.marker").write_text("stub")
+    _seed_data(tmp_path, to_add=[_add_row()], to_update=[_update_row()])
+    run_upload(cfg, dry_run=True, now=NOW)
+    out = capsys.readouterr().out
+    assert "WARNING" in out and "SYNTHETIC" in out
+    assert "would add Wingspan" in out
+    assert "collid 9" in out
+    assert not (tmp_path / "upload_log.csv").exists()
+
+
+# -- run_upload with a fake browser -------------------------------------
+
+
+def test_run_logs_every_attempt_and_continues_past_failures(tmp_path):
+    cfg = _cfg(tmp_path)
+    _seed_data(
+        tmp_path,
+        to_add=[_add_row(bgg_id="1"), _add_row(bgg_id="2", name="Catan")],
+        to_update=[_update_row(collid="9")],
+    )
+    fake = FakeUploader(failures={"Catan"})
+    results = run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
+    assert [r["status"] for r in results] == ["added", "failed", "updated"]
+    logged = list(csv.DictReader((tmp_path / "upload_log.csv").open()))
+    assert len(logged) == 3
+    assert logged[1]["error"] == "RuntimeError: dialog never appeared"
+    assert logged[2]["collid"] == "9"
+
+
+def test_rerun_skips_completed_work(tmp_path):
+    cfg = _cfg(tmp_path)
+    _seed_data(tmp_path, to_add=[_add_row(bgg_id="1"), _add_row(bgg_id="2", name="C")])
+    fake = FakeUploader()
+    run_upload(cfg, uploader=fake, sleep=lambda s: None, now=NOW)
+    assert len(fake.calls) == 2
+    again = FakeUploader()
+    results = run_upload(cfg, uploader=again, sleep=lambda s: None, now=NOW)
+    assert again.calls == [] and results == []
+
+
+def test_pacing_sleeps_2_to_4s_between_games_only(tmp_path):
+    cfg = _cfg(tmp_path)
+    _seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 5)])
+    sleeps: list[float] = []
+    run_upload(
+        cfg,
+        uploader=FakeUploader(),
+        sleep=sleeps.append,
+        rng=random.Random(42),
+        now=NOW,
+    )
+    assert len(sleeps) == 3  # between games, not before the first
+    assert all(2.0 <= s <= 4.0 for s in sleeps)
+
+
+def test_limit_caps_the_queue(tmp_path):
+    cfg = _cfg(tmp_path)
+    _seed_data(tmp_path, to_add=[_add_row(bgg_id=str(i)) for i in range(1, 5)])
+    fake = FakeUploader()
+    run_upload(cfg, uploader=fake, limit=2, sleep=lambda s: None, now=NOW)
+    assert len(fake.calls) == 2
+
+
+# -- credential hygiene -------------------------------------------------
+
+
+def test_scrub_removes_credentials_from_error_text(monkeypatch):
+    monkeypatch.setenv("BGG_USERNAME", "erics-user")
+    monkeypatch.setenv("BGG_PASSWORD", "s3cret-pw")
+    text = 'fill("erics-user") then fill("s3cret-pw") timed out'
+    assert "s3cret-pw" not in _scrub(text)
+    assert "erics-user" not in _scrub(text)
+
+
+# -- verify -------------------------------------------------------------
+
+
+def _item(object_id, coll_id, name="Game", version_id=None):
+    return CollectionItem(
+        object_id=object_id,
+        coll_id=coll_id,
+        name=name,
+        subtype="boardgame",
+        own=True,
+        year=None,
+        version_id=version_id,
+    )
+
+
+def test_verify_flags_missing_and_confirms_present():
+    log = [
+        _log_row(action="add", bgg_id="1", status="added"),
+        _log_row(action="add", bgg_id="2", status="added"),
+        _log_row(
+            action="update", bgg_id="3", collid="30", version_id="7", status="updated"
+        ),
+    ]
+    collection = [_item(1, 10), _item(3, 30, version_id=7)]
+    problems = verify_uploads(log, collection)
+    assert len(problems) == 1
+    assert "not in collection" in problems[0]
+
+
+def test_verify_checks_version_on_adds_and_updates():
+    log = [
+        _log_row(action="add", bgg_id="1", version_id="99", status="added"),
+        _log_row(
+            action="update", bgg_id="3", collid="30", version_id="7", status="updated"
+        ),
+    ]
+    collection = [_item(1, 10, version_id=11), _item(3, 30, version_id=8)]
+    problems = verify_uploads(log, collection)
+    assert len(problems) == 2
+
+
+def test_verify_ignores_failed_rows():
+    log = [_log_row(action="add", bgg_id="5", status="failed")]
+    assert verify_uploads(log, []) == []