Vision backends: any OpenAI-compatible endpoint, including local models

The Anthropic key was the last hard gate for other users. extract's
VisionFn seam gains a second factory speaking the chat-completions
format — OpenAI, OpenRouter, or a local runtime (Ollama, LM Studio,
llama.cpp, vLLM) via config.toml: vision_provider, vision_base_url,
and vision_key_env ("" = keyless local endpoint, no Authorization
header sent). Anthropic stays the default. load_config rejects unknown
providers loudly, the pipeline page's credentials warning follows the
configured provider (a keyless local endpoint warns about nothing),
and config.toml + README document the local-model trade honestly:
weaker spine reading means a longer proofread pass, which the shaky-
read workflow absorbs.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016jXZFSTZQKzAC8fqpWSz9g
This commit is contained in:
Eric Wagoner
2026-08-03 18:26:30 -04:00
co-authored by Claude Fable 5
parent 0cdbf74a02
commit 0358079da4
6 changed files with 173 additions and 3 deletions
+82
View File
@@ -499,3 +499,85 @@ def test_systemic_failures_abort_and_exit_nonzero(tmp_path):
with pytest.raises(typer.Exit):
run_extract(cfg, vision=broken_vision)
assert len(calls) == 3 # aborted after 3 identical failures
# -- pluggable vision backends -------------------------------------------
def test_openai_compatible_vision_speaks_the_format(monkeypatch):
import httpx
from bggpipe.extract import VISION_PROMPT, openai_compatible_vision
captured = {}
def fake_post(url, **kwargs):
captured.update(kwargs, url=url)
request = httpx.Request("POST", url)
return httpx.Response(
200,
request=request,
json={"choices": [{"message": {"content": '{"titles": []}'}}]},
)
monkeypatch.setattr(httpx, "post", fake_post)
monkeypatch.setenv("MY_VISION_KEY", "sk-test")
vision = openai_compatible_vision(
"llava", "http://localhost:11434/v1/", "MY_VISION_KEY"
)
assert vision("QUJD", "image/jpeg") == '{"titles": []}'
assert captured["url"] == "http://localhost:11434/v1/chat/completions"
assert captured["headers"]["Authorization"] == "Bearer sk-test"
body = captured["json"]
assert body["model"] == "llava"
image, prompt = body["messages"][0]["content"]
assert image["image_url"]["url"] == "data:image/jpeg;base64,QUJD"
assert prompt["text"] == VISION_PROMPT
def test_openai_compatible_vision_local_needs_no_key(monkeypatch):
import httpx
from bggpipe.extract import openai_compatible_vision
captured = {}
def fake_post(url, **kwargs):
captured.update(kwargs)
return httpx.Response(
200,
request=httpx.Request("POST", url),
json={"choices": [{"message": {"content": "[]"}}]},
)
monkeypatch.setattr(httpx, "post", fake_post)
vision = openai_compatible_vision("llava", "http://localhost:11434/v1", "")
assert vision("QUJD", "image/jpeg") == "[]"
assert "Authorization" not in captured["headers"]
def test_vision_for_dispatches_by_provider(monkeypatch, tmp_path):
import bggpipe.extract as ex
monkeypatch.setattr(ex, "default_vision", lambda model: f"anthropic:{model}")
cfg = Config(model="claude-sonnet-5")
assert ex.vision_for(cfg) == "anthropic:claude-sonnet-5"
with pytest.raises(ValueError, match="vision_base_url"):
ex.vision_for(Config(vision_provider="openai-compatible"))
fn = ex.vision_for(
Config(
vision_provider="openai-compatible",
vision_base_url="http://localhost:11434/v1",
model="llava",
)
)
assert callable(fn)
def test_config_rejects_unknown_vision_provider(tmp_path):
from bggpipe.config import load_config
p = tmp_path / "config.toml"
p.write_text('vision_provider = "gpt-magic"\n')
with pytest.raises(ValueError, match="vision_provider"):
load_config(p)