diff --git a/README.md b/README.md index f6f8574..9ea65d1 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Every stage is idempotent and resumable: kill it mid-run, restart, lose nothing. ## Requirements - macOS or Linux, Python 3.12+, [uv](https://docs.astral.sh/uv/) -- A vision model for extraction — an [Anthropic API key](https://console.anthropic.com/) by default, or any OpenAI-compatible endpoint via `config.toml` (`vision_provider = "openai-compatible"` + `vision_base_url`): OpenAI, OpenRouter, or a free local runtime like [Ollama](https://ollama.com/) with a vision-capable model. Local models read spines noticeably worse than frontier ones — expect a longer proofread pass on the Titles page, not a broken pipeline. +- A vision model for extraction — an [Anthropic API key](https://console.anthropic.com/) by default, or any OpenAI-compatible endpoint: OpenAI, OpenRouter, or a free local runtime like [Ollama](https://ollama.com/) with a vision-capable model. `config.toml` carries a `[vision.]` block for each; `vision_provider` picks one. Local models read spines noticeably worse than frontier ones — expect a longer proofread pass on the Titles page, not a broken pipeline. - A BoardGameGeek account **and a registered BGG application** — as of BGG's [2025 API policy](https://boardgamegeek.com/using_the_xml_api), the XML API requires a Bearer token from a registered app. Register a free non-commercial application at [boardgamegeek.com/applications](https://boardgamegeek.com/applications) (approval can take a week or more, so **apply on day one**), then create a token. Each user needs their own; tokens must not be shared. ## Quick start diff --git a/config.toml b/config.toml index 9dd5087..298d56b 100644 --- a/config.toml +++ b/config.toml @@ -3,12 +3,19 @@ photos_dir = "photos" data_dir = "data" -model = "claude-sonnet-5" rate_limit_seconds = 2.0 -# Vision backend. The default ("anthropic") reads ANTHROPIC_API_KEY. -# "openai-compatible" covers OpenAI, OpenRouter, and local runtimes: -# vision_provider = "openai-compatible" -# vision_base_url = "http://localhost:11434/v1" # e.g. Ollama -# model = "qwen2.5vl" # a vision-capable model -# vision_key_env = "" # local: no key needed +# Which vision backend reads your shelf photos. Both recipes below stay +# on file; this line picks one. +vision_provider = "anthropic" + +[vision.anthropic] +# reads ANTHROPIC_API_KEY from the environment +model = "claude-sonnet-5" + +[vision."openai-compatible"] +# OpenAI, OpenRouter, or a local runtime (Ollama, LM Studio, vLLM). +# key_env names the env var holding the key; "" = endpoint needs none. +base_url = "http://localhost:11434/v1" +model = "qwen2.5vl:7b" +key_env = "" diff --git a/src/bggpipe/config.py b/src/bggpipe/config.py index c0f2291..5f166e4 100644 --- a/src/bggpipe/config.py +++ b/src/bggpipe/config.py @@ -140,17 +140,39 @@ def load_config(path: Path | None = None) -> Config: "model": str, "rate_limit_seconds": float, "vision_provider": str, - "vision_base_url": str, - "vision_key_env": str, } updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw} - if unknown := sorted(raw.keys() - known.keys()): + if unknown := sorted(raw.keys() - known.keys() - {"vision"}): # a typo'd key silently falling back to defaults is a debugging # trap ("No photos found in photos/") — say so up front warnings.warn( f"{p}: ignoring unknown key(s): {', '.join(unknown)}", stacklevel=2, ) + # [vision.] blocks: every provider's recipe can live in + # the committed file; only the ACTIVE provider's block applies + vision_blocks = raw.get("vision") or {} + if stray := sorted(vision_blocks.keys() - set(VISION_PROVIDERS)): + warnings.warn( + f"{p}: ignoring [vision.*] block(s) for unknown provider(s): " + f"{', '.join(stray)}", + stacklevel=2, + ) + provider = updates.get("vision_provider", cfg.vision_provider) + block = vision_blocks.get(provider) or {} + block_known = { + "model": "model", + "base_url": "vision_base_url", + "key_env": "vision_key_env", + } + if odd := sorted(block.keys() - block_known.keys()): + warnings.warn( + f"{p}: [vision.{provider}]: ignoring unknown key(s): {', '.join(odd)}", + stacklevel=2, + ) + for key, field in block_known.items(): + if key in block: + updates[field] = str(block[key]) cfg = replace(cfg, **updates) if cfg.vision_provider not in VISION_PROVIDERS: # a typo here would surface as a confusing extract failure later diff --git a/src/bggpipe/extract.py b/src/bggpipe/extract.py index 96eda01..3710547 100644 --- a/src/bggpipe/extract.py +++ b/src/bggpipe/extract.py @@ -126,7 +126,12 @@ def parse_vision_response(text: str) -> tuple[list[dict], list[dict], int]: if end == -1: raise ValueError(f"unterminated JSON in vision response: {text[:200]!r}") cleaned = cleaned[start : end + 1] - data = json.loads(cleaned) + try: + data = json.loads(cleaned) + except json.JSONDecodeError: + # local models emit almost-JSON (trailing commas, mostly): one + # cheap repair pass before declaring the response unusable + data = json.loads(re.sub(r",\s*([}\]])", r"\1", cleaned)) if isinstance(data, list): titles_raw, unidentified_raw = data, [] elif isinstance(data, dict): diff --git a/tests/test_config.py b/tests/test_config.py index 884248b..0b997cf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -52,3 +52,47 @@ def test_unknown_toml_keys_warn(tmp_path, monkeypatch): def test_explicit_missing_config_errors_instead_of_silent_defaults(tmp_path): with pytest.raises(FileNotFoundError, match="does not exist"): load_config(tmp_path / "nope.toml") + + +def test_vision_blocks_apply_only_for_the_active_provider(tmp_path): + p = tmp_path / "config.toml" + p.write_text( + """ +vision_provider = "openai-compatible" + +[vision.anthropic] +model = "claude-sonnet-5" + +[vision."openai-compatible"] +base_url = "http://localhost:11434/v1" +model = "qwen2.5vl:7b" +key_env = "" +""" + ) + cfg = load_config(p) + assert cfg.model == "qwen2.5vl:7b" # the active block's model wins + assert cfg.vision_base_url == "http://localhost:11434/v1" + assert cfg.vision_key_env == "" + # flip the provider: the other block applies, this one is inert + p.write_text(p.read_text().replace('"openai-compatible"\n', '"anthropic"\n', 1)) + cfg = load_config(p) + assert cfg.model == "claude-sonnet-5" + assert cfg.vision_base_url == "" # untouched default + + +def test_vision_block_typos_warn(tmp_path): + p = tmp_path / "config.toml" + p.write_text( + """ +[vision.anthropic] +modle = "oops" + +[vision.anthorpic] +model = "claude-sonnet-5" +""" + ) + with ( + pytest.warns(UserWarning, match="modle"), + pytest.warns(UserWarning, match="anthorpic"), + ): + load_config(p) diff --git a/tests/test_extract.py b/tests/test_extract.py index 5de8544..29f386c 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -120,6 +120,15 @@ def test_parse_drops_malformed_entries(): assert len(unidentified) == 1 # empty {} and the bare string are dropped +def test_parse_repairs_trailing_commas(): + # the almost-JSON local vision models emit + titles, unidentified, dropped = parse_vision_response( + '{"titles": [{"title_raw": "Catan",},], "unidentified": [],}' + ) + assert [t["title_raw"] for t in titles] == ["Catan"] + assert unidentified == [] and dropped == 0 + + def test_parse_raises_on_garbage(): with pytest.raises(ValueError): parse_vision_response("I couldn't see any games clearly.")