Re-audit round 4: 5 blind reviewers over the new surface — 24 fixes, +28 tests
The findings clustered exactly where prediction said: the unreviewed web layer. The big ones: decisions made while an extract/resolve job runs are now refused with a 409 (the job's end-of-run rewrite from a start-of-run snapshot would silently revert them); a cross-origin guard blocks preflight-free mutations from hostile webpages (bodyless run triggers, cross-site photo form posts); the JobRunner sets terminal status in a finally catching BaseException (a greenlet death could wedge every future run behind 409s) and writes tracebacks into the visible job log; and a boot token lets clients accept the revision reset after a server restart instead of freezing forever. Even the thrice-audited core yielded one HIGH: an unvetoed bare typo-read sibling of a confident row duplicated its add when the game wasn't in the collection — diff now treats it as satisfied. Second-copy adds carry a flag through to_add.csv and the upload log so verify honestly reports them unverifiable instead of OK. Also: merged_into chains collapse transitively; diff/enrich treat a BGG queue timeout like a missing token; enrich prunes orphaned games.json keys; the wizard shell-quotes .env values and creates the file 0600 from the first byte; fsio stats the tmp inode before replace and uses unique tmp names; an explicit missing --config errors; storage state is owner-only; extract re-extracts corrupt caches, aborts on 3 identical failures, and exits nonzero when nothing succeeded; torn JSON artifacts degrade with in-browser warnings instead of 500ing every page; photo uploads are atomic with cache-invalidation ordered first; the pipeline page computes `running` before the buttons that depend on it; the photo dropzone alerts on network failure; and lost-contact banners clear on recovery everywhere. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Eric Wagoner
@@ -4,6 +4,7 @@ stage functions are injected so nothing slow or networked ever runs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import typer
|
||||
from fastapi.testclient import TestClient
|
||||
@@ -315,3 +316,308 @@ def test_non_photo_files_are_invisible(tmp_path):
|
||||
web = _app(cfg)
|
||||
assert [p["name"] for p in web.get("/api/photos-list").json()] == ["shelf.jpg"]
|
||||
assert web.get("/photos/.DS_Store").status_code == 404
|
||||
|
||||
|
||||
# -- jobs: live-progress and termination contracts ----------------------
|
||||
|
||||
|
||||
def test_running_snapshot_shows_partial_line_then_finishes():
|
||||
import typer as _typer
|
||||
|
||||
runner = JobRunner()
|
||||
release = threading.Event()
|
||||
|
||||
def stage():
|
||||
print("progress: 40%", end="", flush=True) # no newline yet
|
||||
release.wait()
|
||||
_typer.echo(" done")
|
||||
|
||||
runner.start("extract", stage)
|
||||
for _ in range(100):
|
||||
snap = runner.snapshot()
|
||||
if snap["log"]:
|
||||
break
|
||||
time.sleep(0.01)
|
||||
assert snap["status"] == "running"
|
||||
assert snap["log"] == ["progress: 40%"]
|
||||
release.set()
|
||||
runner.wait()
|
||||
assert runner.snapshot()["log"] == ["progress: 40% done"]
|
||||
|
||||
|
||||
def test_log_serves_last_200_lines_and_buffer_is_bounded():
|
||||
import typer as _typer
|
||||
|
||||
from bggpipe.jobs import MAX_LOG_LINES
|
||||
|
||||
runner = JobRunner()
|
||||
|
||||
def stage():
|
||||
for i in range(MAX_LOG_LINES + 300):
|
||||
_typer.echo(f"line {i}")
|
||||
|
||||
runner.start("extract", stage)
|
||||
runner.wait()
|
||||
log = runner.snapshot()["log"]
|
||||
assert len(log) == 200
|
||||
assert log[-1] == f"line {MAX_LOG_LINES + 299}"
|
||||
|
||||
|
||||
def test_zero_exit_codes_count_as_done():
|
||||
import typer as _typer
|
||||
|
||||
for exc in (_typer.Exit(), SystemExit(0)):
|
||||
runner = JobRunner()
|
||||
runner.start("diff", lambda exc=exc: (_ for _ in ()).throw(exc))
|
||||
runner.wait()
|
||||
assert runner.snapshot()["status"] == "done"
|
||||
|
||||
|
||||
def test_nonzero_systemexit_is_failed():
|
||||
runner = JobRunner()
|
||||
runner.start("diff", lambda: (_ for _ in ()).throw(SystemExit(2)))
|
||||
runner.wait()
|
||||
snap = runner.snapshot()
|
||||
assert snap["status"] == "failed" and "2" in snap["error"]
|
||||
|
||||
|
||||
def test_base_exception_cannot_wedge_the_runner():
|
||||
class Rude(BaseException):
|
||||
pass
|
||||
|
||||
runner = JobRunner()
|
||||
runner.start("upload", lambda: (_ for _ in ()).throw(Rude("greenlet died")))
|
||||
runner.wait()
|
||||
snap = runner.snapshot()
|
||||
assert snap["status"] == "failed"
|
||||
assert "Rude" in snap["error"]
|
||||
assert any("Traceback" in line for line in snap["log"]) # diagnosable
|
||||
assert runner.start("extract", lambda: None) # slot is free again
|
||||
runner.wait()
|
||||
|
||||
|
||||
# -- concurrency and restart contracts ----------------------------------
|
||||
|
||||
|
||||
def test_revision_bumps_on_mutations_not_reads(tmp_path):
|
||||
from bggpipe.resolve import write_matches
|
||||
|
||||
cfg = _cfg(tmp_path)
|
||||
write_matches(
|
||||
cfg.matches_path,
|
||||
[
|
||||
{
|
||||
"title_raw": "Mystery",
|
||||
"bgg_id": "",
|
||||
"bgg_name": "",
|
||||
"year": "",
|
||||
"type": "",
|
||||
"match_status": "unmatched",
|
||||
"version_id": "",
|
||||
"version_name": "",
|
||||
"version_status": "",
|
||||
"candidates_json": "[]",
|
||||
"version_candidates_json": "[]",
|
||||
"source_photos": "x.jpg",
|
||||
}
|
||||
],
|
||||
)
|
||||
web = _app(cfg)
|
||||
first = web.get("/api/state").json()
|
||||
assert "boot" in first # clients detect restarts by boot change
|
||||
second = web.get("/api/state").json()
|
||||
assert second["revision"] == first["revision"] # reads never bump
|
||||
|
||||
res = web.post(
|
||||
"/api/decision",
|
||||
json={"title_raw": "Mystery", "source_photos": "x.jpg", "action": "reject"},
|
||||
)
|
||||
assert res.json()["revision"] > first["revision"] # mutations bump
|
||||
|
||||
# external rewrite bumps on the next read
|
||||
rows = []
|
||||
write_matches(cfg.matches_path, rows)
|
||||
bumped = web.get("/api/state").json()
|
||||
assert bumped["revision"] > res.json()["revision"]
|
||||
|
||||
|
||||
def test_torn_data_files_degrade_with_warnings_not_500(tmp_path):
|
||||
cfg = _cfg(tmp_path)
|
||||
cfg.unidentified_path.write_text("{torn")
|
||||
cfg.games_path.write_text("[not even close")
|
||||
web = _app(cfg)
|
||||
state = web.get("/api/state")
|
||||
pipeline = web.get("/api/pipeline")
|
||||
library = web.get("/api/library")
|
||||
assert state.status_code == pipeline.status_code == library.status_code == 200
|
||||
assert library.json() == []
|
||||
assert any("unreadable" in w for w in state.json()["warnings"])
|
||||
|
||||
|
||||
def test_decisions_locked_out_while_matches_rewriting_job_runs(tmp_path):
|
||||
from bggpipe.resolve import read_matches, write_matches
|
||||
|
||||
cfg = _cfg(tmp_path)
|
||||
write_matches(
|
||||
cfg.matches_path,
|
||||
[
|
||||
{
|
||||
"title_raw": "Mystery",
|
||||
"bgg_id": "",
|
||||
"bgg_name": "",
|
||||
"year": "",
|
||||
"type": "",
|
||||
"match_status": "unmatched",
|
||||
"version_id": "",
|
||||
"version_name": "",
|
||||
"version_status": "",
|
||||
"candidates_json": "[]",
|
||||
"version_candidates_json": "[]",
|
||||
"source_photos": "x.jpg",
|
||||
}
|
||||
],
|
||||
)
|
||||
release = threading.Event()
|
||||
jobs = JobRunner()
|
||||
web = _app(cfg, stages={"resolve": release.wait}, jobs=jobs)
|
||||
web.post("/api/run/resolve")
|
||||
res = web.post(
|
||||
"/api/decision",
|
||||
json={"title_raw": "Mystery", "source_photos": "x.jpg", "action": "reject"},
|
||||
)
|
||||
assert res.status_code == 409
|
||||
assert "resolve run is rewriting" in res.json()["detail"]
|
||||
# the decision was NOT applied
|
||||
assert read_matches(cfg.matches_path)[0]["match_status"] == "unmatched"
|
||||
release.set()
|
||||
jobs.wait()
|
||||
|
||||
|
||||
def test_cross_origin_mutations_are_refused(tmp_path):
|
||||
web = _app(_cfg(tmp_path))
|
||||
# foreign Origin on a same-host request: refused
|
||||
res = web.post("/api/run/extract", headers={"origin": "https://evil.example"})
|
||||
assert res.status_code == 403
|
||||
# foreign Host (DNS rebinding): refused
|
||||
res = web.post(
|
||||
"/api/photos",
|
||||
headers={"host": "evil.example"},
|
||||
files={"files": ("a.jpg", b"x", "image/jpeg")},
|
||||
)
|
||||
assert res.status_code == 403
|
||||
# reads are unaffected
|
||||
assert (
|
||||
web.get("/api/pipeline", headers={"origin": "https://evil.example"}).status_code
|
||||
== 200
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_decisions_and_polls_never_lose_a_decision(tmp_path):
|
||||
from bggpipe.resolve import read_matches, write_matches
|
||||
|
||||
cfg = _cfg(tmp_path)
|
||||
base = {
|
||||
"bgg_id": "",
|
||||
"bgg_name": "",
|
||||
"year": "",
|
||||
"type": "",
|
||||
"match_status": "unmatched",
|
||||
"version_id": "",
|
||||
"version_name": "",
|
||||
"version_status": "",
|
||||
"candidates_json": "[]",
|
||||
"version_candidates_json": "[]",
|
||||
"source_photos": "x.jpg",
|
||||
}
|
||||
rows = [{**base, "title_raw": f"Game{i}"} for i in range(8)]
|
||||
write_matches(cfg.matches_path, rows)
|
||||
web = _app(cfg)
|
||||
|
||||
def decide(i):
|
||||
web.post(
|
||||
"/api/decision",
|
||||
json={
|
||||
"title_raw": f"Game{i}",
|
||||
"source_photos": "x.jpg",
|
||||
"action": "reject",
|
||||
},
|
||||
)
|
||||
|
||||
def poll():
|
||||
web.get("/api/state")
|
||||
|
||||
threads = [threading.Thread(target=decide, args=(i,)) for i in range(8)]
|
||||
threads += [threading.Thread(target=poll) for _ in range(8)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
saved = read_matches(cfg.matches_path)
|
||||
assert all(r["match_status"] == "rejected" for r in saved)
|
||||
|
||||
|
||||
def test_decision_for_vanished_row_is_404_and_touches_nothing(tmp_path):
|
||||
from bggpipe.resolve import read_matches, write_matches
|
||||
|
||||
cfg = _cfg(tmp_path)
|
||||
row = {
|
||||
"title_raw": "Kept",
|
||||
"bgg_id": "",
|
||||
"bgg_name": "",
|
||||
"year": "",
|
||||
"type": "",
|
||||
"match_status": "unmatched",
|
||||
"version_id": "",
|
||||
"version_name": "",
|
||||
"version_status": "",
|
||||
"candidates_json": "[]",
|
||||
"version_candidates_json": "[]",
|
||||
"source_photos": "x.jpg",
|
||||
}
|
||||
write_matches(cfg.matches_path, [row])
|
||||
web = _app(cfg)
|
||||
res = web.post(
|
||||
"/api/decision",
|
||||
json={"title_raw": "Gone", "source_photos": "x.jpg", "action": "reject"},
|
||||
)
|
||||
assert res.status_code == 404
|
||||
assert read_matches(cfg.matches_path)[0]["match_status"] == "unmatched"
|
||||
|
||||
|
||||
def test_pipeline_reports_badge_fields(tmp_path):
|
||||
from bggpipe.resolve import write_matches
|
||||
|
||||
cfg = _cfg(tmp_path)
|
||||
base = {
|
||||
"bgg_id": "13",
|
||||
"bgg_name": "Catan",
|
||||
"year": "",
|
||||
"type": "boardgame",
|
||||
"version_id": "",
|
||||
"version_name": "",
|
||||
"candidates_json": "[]",
|
||||
"version_candidates_json": "[]",
|
||||
"source_photos": "x.jpg",
|
||||
}
|
||||
write_matches(
|
||||
cfg.matches_path,
|
||||
[
|
||||
{
|
||||
**base,
|
||||
"title_raw": "A",
|
||||
"match_status": "ambiguous",
|
||||
"version_status": "",
|
||||
},
|
||||
{
|
||||
**base,
|
||||
"title_raw": "B",
|
||||
"match_status": "auto",
|
||||
"version_status": "version_ambiguous",
|
||||
"version_id": "1",
|
||||
},
|
||||
],
|
||||
)
|
||||
cfg.to_add_path.write_text("bgg_id,bgg_name\n1,X\n2,Y\n")
|
||||
p = _app(cfg).get("/api/pipeline").json()
|
||||
assert p["pending_review"] == 2 # one match + one edition decision
|
||||
assert p["to_add"] == 2 # header excluded
|
||||
|
||||
Reference in New Issue
Block a user