Vision backends: any OpenAI-compatible endpoint, including local models

The Anthropic key was the last hard gate for other users. extract's
VisionFn seam gains a second factory speaking the chat-completions
format — OpenAI, OpenRouter, or a local runtime (Ollama, LM Studio,
llama.cpp, vLLM) via config.toml: vision_provider, vision_base_url,
and vision_key_env ("" = keyless local endpoint, no Authorization
header sent). Anthropic stays the default. load_config rejects unknown
providers loudly, the pipeline page's credentials warning follows the
configured provider (a keyless local endpoint warns about nothing),
and config.toml + README document the local-model trade honestly:
weaker spine reading means a longer proofread pass, which the shaky-
read workflow absorbs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016jXZFSTZQKzAC8fqpWSz9g

README.md

@@ -70,7 +70,7 @@ Every stage is idempotent and resumable: kill it mid-run, restart, lose nothing.
 ## Requirements
 
 - macOS or Linux, Python 3.12+, [uv](https://docs.astral.sh/uv/)
-- An [Anthropic API key](https://console.anthropic.com/) (vision extraction)
+- A vision model for extraction — an [Anthropic API key](https://console.anthropic.com/) by default, or any OpenAI-compatible endpoint via `config.toml` (`vision_provider = "openai-compatible"` + `vision_base_url`): OpenAI, OpenRouter, or a free local runtime like [Ollama](https://ollama.com/) with a vision-capable model. Local models read spines noticeably worse than frontier ones — expect a longer proofread pass on the Titles page, not a broken pipeline.
 - A BoardGameGeek account **and a registered BGG application** — as of BGG's [2025 API policy](https://boardgamegeek.com/using_the_xml_api), the XML API requires a Bearer token from a registered app. Register a free non-commercial application at [boardgamegeek.com/applications](https://boardgamegeek.com/applications) (approval can take a week or more, so **apply on day one**), then create a token. Each user needs their own; tokens must not be shared.
 
 ## Quick start

config.toml

@@ -5,3 +5,10 @@ photos_dir = "photos"
 data_dir = "data"
 model = "claude-sonnet-5"
 rate_limit_seconds = 2.0
+
+# Vision backend. The default ("anthropic") reads ANTHROPIC_API_KEY.
+# "openai-compatible" covers OpenAI, OpenRouter, and local runtimes:
+#   vision_provider = "openai-compatible"
+#   vision_base_url = "http://localhost:11434/v1"   # e.g. Ollama
+#   model = "qwen2.5vl"                             # a vision-capable model
+#   vision_key_env = ""                             # local: no key needed

src/bggpipe/config.py

@@ -22,6 +22,9 @@ STUB_CACHE_MARKER_NAME = "STUB_FIXTURES.marker"
 STUB_DATA_MARKER_NAME = "STUB_DATA.marker"
 
 
+VISION_PROVIDERS = ("anthropic", "openai-compatible")
+
+
 @dataclass(frozen=True)
 class Config:
     bgg_username: str = ""
@@ -29,6 +32,11 @@ class Config:
     data_dir: Path = Path("data")
     model: str = "claude-sonnet-5"
     rate_limit_seconds: float = 2.0
+    # "anthropic" (default) or "openai-compatible" — the latter covers
+    # OpenAI, OpenRouter, and local runtimes (Ollama, LM Studio, vLLM)
+    vision_provider: str = "anthropic"
+    vision_base_url: str = ""  # e.g. http://localhost:11434/v1 for Ollama
+    vision_key_env: str = "OPENAI_API_KEY"  # "" = endpoint needs no key
 
     @property
     def cache_dir(self) -> Path:
@@ -131,6 +139,9 @@ def load_config(path: Path | None = None) -> Config:
             "data_dir": Path,
             "model": str,
             "rate_limit_seconds": float,
+            "vision_provider": str,
+            "vision_base_url": str,
+            "vision_key_env": str,
         }
         updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw}
         if unknown := sorted(raw.keys() - known.keys()):
@@ -141,6 +152,12 @@ def load_config(path: Path | None = None) -> Config:
                 stacklevel=2,
             )
         cfg = replace(cfg, **updates)
+    if cfg.vision_provider not in VISION_PROVIDERS:
+        # a typo here would surface as a confusing extract failure later
+        raise ValueError(
+            f"{p}: vision_provider must be one of {', '.join(VISION_PROVIDERS)}"
+            f" (got {cfg.vision_provider!r})"
+        )
     if username := os.environ.get("BGG_USERNAME"):
         cfg = replace(cfg, bgg_username=username)
     return cfg

src/bggpipe/extract.py

@@ -12,6 +12,7 @@ from __future__ import annotations
 import base64
 import io
 import json
+import os
 import re
 from collections.abc import Callable
 from pathlib import Path
@@ -176,6 +177,61 @@ def default_vision(model: str) -> VisionFn:
     return vision
 
 
+def openai_compatible_vision(model: str, base_url: str, key_env: str) -> VisionFn:
+    """Any endpoint speaking the OpenAI chat-completions format: OpenAI
+    itself, OpenRouter, or a local runtime (Ollama, LM Studio, llama.cpp,
+    vLLM — pass its /v1 base URL). A key is optional: local runtimes run
+    without one, so an empty key_env just sends no Authorization header."""
+    import httpx
+
+    headers = {}
+    if key_env and (key := os.environ.get(key_env)):
+        headers["Authorization"] = f"Bearer {key}"
+
+    def vision(image_b64: str, media_type: str) -> str:
+        response = httpx.post(
+            base_url.rstrip("/") + "/chat/completions",
+            headers=headers,
+            json={
+                "model": model,
+                "max_tokens": 4000,
+                "messages": [
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": f"data:{media_type};base64,{image_b64}"
+                                },
+                            },
+                            {"type": "text", "text": VISION_PROMPT},
+                        ],
+                    }
+                ],
+            },
+            timeout=300.0,  # local models on modest hardware are slow
+        )
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"] or ""
+
+    return vision
+
+
+def vision_for(cfg: Config) -> VisionFn:
+    """The configured vision backend (config.toml: vision_provider)."""
+    if cfg.vision_provider == "openai-compatible":
+        if not cfg.vision_base_url:
+            raise ValueError(
+                "vision_provider 'openai-compatible' needs vision_base_url "
+                "in config.toml (e.g. http://localhost:11434/v1 for Ollama)"
+            )
+        return openai_compatible_vision(
+            cfg.model, cfg.vision_base_url, cfg.vision_key_env
+        )
+    return default_vision(cfg.model)
+
+
 def extract_photo(photo: Path, vision: VisionFn) -> dict:
     """One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache
     file format)."""
@@ -536,7 +592,7 @@ def run_extract(
 
     raw_dir = cfg.extract_raw_dir
     raw_dir.mkdir(parents=True, exist_ok=True)
-    vision = vision or default_vision(cfg.model)
+    vision = vision or vision_for(cfg)
 
     failed: list[str] = []
     consecutive: tuple[str, int] = ("", 0)

src/bggpipe/webreview.py

@@ -758,7 +758,15 @@ def create_app(
                 "env": {
                     key: bool(os.environ.get(key))
                     for key in (
-                        "ANTHROPIC_API_KEY",
+                        # the vision key follows the configured provider;
+                        # a keyless local endpoint warns about nothing
+                        *(
+                            ["ANTHROPIC_API_KEY"]
+                            if cfg.vision_provider == "anthropic"
+                            else [cfg.vision_key_env]
+                            if cfg.vision_key_env
+                            else []
+                        ),
                         "BGG_API_TOKEN",
                         "BGG_USERNAME",
                         "BGG_PASSWORD",

tests/test_extract.py

@@ -499,3 +499,85 @@ def test_systemic_failures_abort_and_exit_nonzero(tmp_path):
     with pytest.raises(typer.Exit):
         run_extract(cfg, vision=broken_vision)
     assert len(calls) == 3  # aborted after 3 identical failures
+
+
+# -- pluggable vision backends -------------------------------------------
+
+
+def test_openai_compatible_vision_speaks_the_format(monkeypatch):
+    import httpx
+
+    from bggpipe.extract import VISION_PROMPT, openai_compatible_vision
+
+    captured = {}
+
+    def fake_post(url, **kwargs):
+        captured.update(kwargs, url=url)
+        request = httpx.Request("POST", url)
+        return httpx.Response(
+            200,
+            request=request,
+            json={"choices": [{"message": {"content": '{"titles": []}'}}]},
+        )
+
+    monkeypatch.setattr(httpx, "post", fake_post)
+    monkeypatch.setenv("MY_VISION_KEY", "sk-test")
+    vision = openai_compatible_vision(
+        "llava", "http://localhost:11434/v1/", "MY_VISION_KEY"
+    )
+    assert vision("QUJD", "image/jpeg") == '{"titles": []}'
+    assert captured["url"] == "http://localhost:11434/v1/chat/completions"
+    assert captured["headers"]["Authorization"] == "Bearer sk-test"
+    body = captured["json"]
+    assert body["model"] == "llava"
+    image, prompt = body["messages"][0]["content"]
+    assert image["image_url"]["url"] == "data:image/jpeg;base64,QUJD"
+    assert prompt["text"] == VISION_PROMPT
+
+
+def test_openai_compatible_vision_local_needs_no_key(monkeypatch):
+    import httpx
+
+    from bggpipe.extract import openai_compatible_vision
+
+    captured = {}
+
+    def fake_post(url, **kwargs):
+        captured.update(kwargs)
+        return httpx.Response(
+            200,
+            request=httpx.Request("POST", url),
+            json={"choices": [{"message": {"content": "[]"}}]},
+        )
+
+    monkeypatch.setattr(httpx, "post", fake_post)
+    vision = openai_compatible_vision("llava", "http://localhost:11434/v1", "")
+    assert vision("QUJD", "image/jpeg") == "[]"
+    assert "Authorization" not in captured["headers"]
+
+
+def test_vision_for_dispatches_by_provider(monkeypatch, tmp_path):
+    import bggpipe.extract as ex
+
+    monkeypatch.setattr(ex, "default_vision", lambda model: f"anthropic:{model}")
+    cfg = Config(model="claude-sonnet-5")
+    assert ex.vision_for(cfg) == "anthropic:claude-sonnet-5"
+    with pytest.raises(ValueError, match="vision_base_url"):
+        ex.vision_for(Config(vision_provider="openai-compatible"))
+    fn = ex.vision_for(
+        Config(
+            vision_provider="openai-compatible",
+            vision_base_url="http://localhost:11434/v1",
+            model="llava",
+        )
+    )
+    assert callable(fn)
+
+
+def test_config_rejects_unknown_vision_provider(tmp_path):
+    from bggpipe.config import load_config
+
+    p = tmp_path / "config.toml"
+    p.write_text('vision_provider = "gpt-magic"\n')
+    with pytest.raises(ValueError, match="vision_provider"):
+        load_config(p)