From 0358079da4036a0a71288c246f66d93e5e8ad1b1 Mon Sep 17 00:00:00 2001 From: Eric Wagoner Date: Mon, 3 Aug 2026 18:26:30 -0400 Subject: [PATCH] Vision backends: any OpenAI-compatible endpoint, including local models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Anthropic key was the last hard gate for other users. extract's VisionFn seam gains a second factory speaking the chat-completions format — OpenAI, OpenRouter, or a local runtime (Ollama, LM Studio, llama.cpp, vLLM) via config.toml: vision_provider, vision_base_url, and vision_key_env ("" = keyless local endpoint, no Authorization header sent). Anthropic stays the default. load_config rejects unknown providers loudly, the pipeline page's credentials warning follows the configured provider (a keyless local endpoint warns about nothing), and config.toml + README document the local-model trade honestly: weaker spine reading means a longer proofread pass, which the shaky- read workflow absorbs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016jXZFSTZQKzAC8fqpWSz9g --- README.md | 2 +- config.toml | 7 ++++ src/bggpipe/config.py | 17 +++++++++ src/bggpipe/extract.py | 58 +++++++++++++++++++++++++++- src/bggpipe/webreview.py | 10 ++++- tests/test_extract.py | 82 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 173 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9231d4f..f6f8574 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Every stage is idempotent and resumable: kill it mid-run, restart, lose nothing. ## Requirements - macOS or Linux, Python 3.12+, [uv](https://docs.astral.sh/uv/) -- An [Anthropic API key](https://console.anthropic.com/) (vision extraction) +- A vision model for extraction — an [Anthropic API key](https://console.anthropic.com/) by default, or any OpenAI-compatible endpoint via `config.toml` (`vision_provider = "openai-compatible"` + `vision_base_url`): OpenAI, OpenRouter, or a free local runtime like [Ollama](https://ollama.com/) with a vision-capable model. Local models read spines noticeably worse than frontier ones — expect a longer proofread pass on the Titles page, not a broken pipeline. - A BoardGameGeek account **and a registered BGG application** — as of BGG's [2025 API policy](https://boardgamegeek.com/using_the_xml_api), the XML API requires a Bearer token from a registered app. Register a free non-commercial application at [boardgamegeek.com/applications](https://boardgamegeek.com/applications) (approval can take a week or more, so **apply on day one**), then create a token. Each user needs their own; tokens must not be shared. ## Quick start diff --git a/config.toml b/config.toml index fe3d556..9dd5087 100644 --- a/config.toml +++ b/config.toml @@ -5,3 +5,10 @@ photos_dir = "photos" data_dir = "data" model = "claude-sonnet-5" rate_limit_seconds = 2.0 + +# Vision backend. The default ("anthropic") reads ANTHROPIC_API_KEY. +# "openai-compatible" covers OpenAI, OpenRouter, and local runtimes: +# vision_provider = "openai-compatible" +# vision_base_url = "http://localhost:11434/v1" # e.g. Ollama +# model = "qwen2.5vl" # a vision-capable model +# vision_key_env = "" # local: no key needed diff --git a/src/bggpipe/config.py b/src/bggpipe/config.py index 20c75ad..c0f2291 100644 --- a/src/bggpipe/config.py +++ b/src/bggpipe/config.py @@ -22,6 +22,9 @@ STUB_CACHE_MARKER_NAME = "STUB_FIXTURES.marker" STUB_DATA_MARKER_NAME = "STUB_DATA.marker" +VISION_PROVIDERS = ("anthropic", "openai-compatible") + + @dataclass(frozen=True) class Config: bgg_username: str = "" @@ -29,6 +32,11 @@ class Config: data_dir: Path = Path("data") model: str = "claude-sonnet-5" rate_limit_seconds: float = 2.0 + # "anthropic" (default) or "openai-compatible" — the latter covers + # OpenAI, OpenRouter, and local runtimes (Ollama, LM Studio, vLLM) + vision_provider: str = "anthropic" + vision_base_url: str = "" # e.g. http://localhost:11434/v1 for Ollama + vision_key_env: str = "OPENAI_API_KEY" # "" = endpoint needs no key @property def cache_dir(self) -> Path: @@ -131,6 +139,9 @@ def load_config(path: Path | None = None) -> Config: "data_dir": Path, "model": str, "rate_limit_seconds": float, + "vision_provider": str, + "vision_base_url": str, + "vision_key_env": str, } updates = {key: caster(raw[key]) for key, caster in known.items() if key in raw} if unknown := sorted(raw.keys() - known.keys()): @@ -141,6 +152,12 @@ def load_config(path: Path | None = None) -> Config: stacklevel=2, ) cfg = replace(cfg, **updates) + if cfg.vision_provider not in VISION_PROVIDERS: + # a typo here would surface as a confusing extract failure later + raise ValueError( + f"{p}: vision_provider must be one of {', '.join(VISION_PROVIDERS)}" + f" (got {cfg.vision_provider!r})" + ) if username := os.environ.get("BGG_USERNAME"): cfg = replace(cfg, bgg_username=username) return cfg diff --git a/src/bggpipe/extract.py b/src/bggpipe/extract.py index 4b8c295..96eda01 100644 --- a/src/bggpipe/extract.py +++ b/src/bggpipe/extract.py @@ -12,6 +12,7 @@ from __future__ import annotations import base64 import io import json +import os import re from collections.abc import Callable from pathlib import Path @@ -176,6 +177,61 @@ def default_vision(model: str) -> VisionFn: return vision +def openai_compatible_vision(model: str, base_url: str, key_env: str) -> VisionFn: + """Any endpoint speaking the OpenAI chat-completions format: OpenAI + itself, OpenRouter, or a local runtime (Ollama, LM Studio, llama.cpp, + vLLM — pass its /v1 base URL). A key is optional: local runtimes run + without one, so an empty key_env just sends no Authorization header.""" + import httpx + + headers = {} + if key_env and (key := os.environ.get(key_env)): + headers["Authorization"] = f"Bearer {key}" + + def vision(image_b64: str, media_type: str) -> str: + response = httpx.post( + base_url.rstrip("/") + "/chat/completions", + headers=headers, + json={ + "model": model, + "max_tokens": 4000, + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:{media_type};base64,{image_b64}" + }, + }, + {"type": "text", "text": VISION_PROMPT}, + ], + } + ], + }, + timeout=300.0, # local models on modest hardware are slow + ) + response.raise_for_status() + return response.json()["choices"][0]["message"]["content"] or "" + + return vision + + +def vision_for(cfg: Config) -> VisionFn: + """The configured vision backend (config.toml: vision_provider).""" + if cfg.vision_provider == "openai-compatible": + if not cfg.vision_base_url: + raise ValueError( + "vision_provider 'openai-compatible' needs vision_base_url " + "in config.toml (e.g. http://localhost:11434/v1 for Ollama)" + ) + return openai_compatible_vision( + cfg.model, cfg.vision_base_url, cfg.vision_key_env + ) + return default_vision(cfg.model) + + def extract_photo(photo: Path, vision: VisionFn) -> dict: """One photo -> {"titles": [...], "unidentified": [...]} (the raw-cache file format).""" @@ -536,7 +592,7 @@ def run_extract( raw_dir = cfg.extract_raw_dir raw_dir.mkdir(parents=True, exist_ok=True) - vision = vision or default_vision(cfg.model) + vision = vision or vision_for(cfg) failed: list[str] = [] consecutive: tuple[str, int] = ("", 0) diff --git a/src/bggpipe/webreview.py b/src/bggpipe/webreview.py index 7a4ccd7..7d1b93a 100644 --- a/src/bggpipe/webreview.py +++ b/src/bggpipe/webreview.py @@ -758,7 +758,15 @@ def create_app( "env": { key: bool(os.environ.get(key)) for key in ( - "ANTHROPIC_API_KEY", + # the vision key follows the configured provider; + # a keyless local endpoint warns about nothing + *( + ["ANTHROPIC_API_KEY"] + if cfg.vision_provider == "anthropic" + else [cfg.vision_key_env] + if cfg.vision_key_env + else [] + ), "BGG_API_TOKEN", "BGG_USERNAME", "BGG_PASSWORD", diff --git a/tests/test_extract.py b/tests/test_extract.py index efc9c3d..5de8544 100644 --- a/tests/test_extract.py +++ b/tests/test_extract.py @@ -499,3 +499,85 @@ def test_systemic_failures_abort_and_exit_nonzero(tmp_path): with pytest.raises(typer.Exit): run_extract(cfg, vision=broken_vision) assert len(calls) == 3 # aborted after 3 identical failures + + +# -- pluggable vision backends ------------------------------------------- + + +def test_openai_compatible_vision_speaks_the_format(monkeypatch): + import httpx + + from bggpipe.extract import VISION_PROMPT, openai_compatible_vision + + captured = {} + + def fake_post(url, **kwargs): + captured.update(kwargs, url=url) + request = httpx.Request("POST", url) + return httpx.Response( + 200, + request=request, + json={"choices": [{"message": {"content": '{"titles": []}'}}]}, + ) + + monkeypatch.setattr(httpx, "post", fake_post) + monkeypatch.setenv("MY_VISION_KEY", "sk-test") + vision = openai_compatible_vision( + "llava", "http://localhost:11434/v1/", "MY_VISION_KEY" + ) + assert vision("QUJD", "image/jpeg") == '{"titles": []}' + assert captured["url"] == "http://localhost:11434/v1/chat/completions" + assert captured["headers"]["Authorization"] == "Bearer sk-test" + body = captured["json"] + assert body["model"] == "llava" + image, prompt = body["messages"][0]["content"] + assert image["image_url"]["url"] == "data:image/jpeg;base64,QUJD" + assert prompt["text"] == VISION_PROMPT + + +def test_openai_compatible_vision_local_needs_no_key(monkeypatch): + import httpx + + from bggpipe.extract import openai_compatible_vision + + captured = {} + + def fake_post(url, **kwargs): + captured.update(kwargs) + return httpx.Response( + 200, + request=httpx.Request("POST", url), + json={"choices": [{"message": {"content": "[]"}}]}, + ) + + monkeypatch.setattr(httpx, "post", fake_post) + vision = openai_compatible_vision("llava", "http://localhost:11434/v1", "") + assert vision("QUJD", "image/jpeg") == "[]" + assert "Authorization" not in captured["headers"] + + +def test_vision_for_dispatches_by_provider(monkeypatch, tmp_path): + import bggpipe.extract as ex + + monkeypatch.setattr(ex, "default_vision", lambda model: f"anthropic:{model}") + cfg = Config(model="claude-sonnet-5") + assert ex.vision_for(cfg) == "anthropic:claude-sonnet-5" + with pytest.raises(ValueError, match="vision_base_url"): + ex.vision_for(Config(vision_provider="openai-compatible")) + fn = ex.vision_for( + Config( + vision_provider="openai-compatible", + vision_base_url="http://localhost:11434/v1", + model="llava", + ) + ) + assert callable(fn) + + +def test_config_rejects_unknown_vision_provider(tmp_path): + from bggpipe.config import load_config + + p = tmp_path / "config.toml" + p.write_text('vision_provider = "gpt-magic"\n') + with pytest.raises(ValueError, match="vision_provider"): + load_config(p)