From dd8ebb2d38c8c8e25c592d5186389311de0703f5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 18:12:41 +0000 Subject: [PATCH 1/7] Generalize LLM post-processing to any OpenAI-API-compatible server Rename the settings tab from "Ollama" to "OpenAI API" and switch the LLM client from Ollama's native API to the OpenAI-compatible /v1/models and /v1/chat/completions endpoints, so any compatible server (Ollama, LM Studio, OpenAI, etc.) can be used. The server URL still defaults to localhost but remains user-editable, and a new optional API key field is sent as an Authorization bearer token for non-local servers. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01PThnjsJtj5TRZNBzCLECoi --- Cargo.lock | 1 + crates/voxctrl-config/src/lib.rs | 5 ++ crates/voxctrl-llm/src/lib.rs | 102 ++++++++++++++++++++---------- docs/api.md | 10 +-- docs/configuration.md | 10 ++- src-tauri/src/commands.rs | 6 +- src/lib/Settings/OllamaTab.svelte | 11 +++- src/lib/Settings/Settings.svelte | 2 +- src/stores/config.ts | 2 + tests/svelte/Settings.test.ts | 1 + 10 files changed, 106 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b9260b..10f84b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9237,6 +9237,7 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2088172d00f936c348d6a72f488dc2660ab3f507263a195df308a3c2383229f6" dependencies = [ + "libc", "whisper-rs-sys", ] diff --git a/crates/voxctrl-config/src/lib.rs b/crates/voxctrl-config/src/lib.rs index cdef4e4..1c3e659 100644 --- a/crates/voxctrl-config/src/lib.rs +++ b/crates/voxctrl-config/src/lib.rs @@ -230,7 +230,11 @@ pub struct OllamaConfig { pub mode: OllamaMode, /// Used when mode == Custom. "{text}" is substituted. pub custom_prompt: Option, + /// Base URL of any OpenAI-API-compatible server (Ollama, LM Studio, OpenAI, etc.) pub endpoint: String, + /// Optional bearer token, sent as `Authorization: Bearer ` when set. + #[serde(default)] + pub api_key: Option, pub timeout_secs: u64, } @@ -242,6 +246,7 @@ impl Default for OllamaConfig { mode: OllamaMode::Clean, custom_prompt: None, endpoint: "http://localhost:11434".into(), + api_key: None, timeout_secs: 30, } } diff --git a/crates/voxctrl-llm/src/lib.rs b/crates/voxctrl-llm/src/lib.rs index da49f7d..3af5ad1 100644 --- a/crates/voxctrl-llm/src/lib.rs +++ b/crates/voxctrl-llm/src/lib.rs @@ -27,18 +27,34 @@ fn mode_prompt(mode: &OllamaMode, text: &str) -> String { } } -// ── Ollama API types ────────────────────────────────────────────────────────── +// ── OpenAI-compatible API types ──────────────────────────────────────────────── #[derive(Serialize)] -struct GenerateRequest<'a> { +struct ChatMessage<'a> { + role: &'a str, + content: &'a str, +} + +#[derive(Serialize)] +struct ChatCompletionRequest<'a> { model: &'a str, - prompt: &'a str, + messages: Vec>, stream: bool, } #[derive(Deserialize)] -struct GenerateResponse { - response: String, +struct ChatCompletionChoice { + message: ChatCompletionMessage, +} + +#[derive(Deserialize)] +struct ChatCompletionMessage { + content: String, +} + +#[derive(Deserialize)] +struct ChatCompletionResponse { + choices: Vec, } // ── Client ──────────────────────────────────────────────────────────────────── @@ -63,7 +79,15 @@ impl OllamaClient { } } - /// Lazily probe if Ollama is reachable. Cached after first check. + fn auth_header(&self) -> Option { + self.config + .api_key + .as_ref() + .filter(|k| !k.is_empty()) + .map(|k| format!("Bearer {k}")) + } + + /// Lazily probe if the server is reachable. Cached after first check. pub async fn is_available(&self) -> bool { { let guard = self.available.lock().unwrap(); @@ -71,25 +95,26 @@ impl OllamaClient { return v; } } - let url = format!("{}/api/tags", self.config.endpoint); - let ok = self - .http - .get(&url) - .timeout(Duration::from_secs(2)) + let url = format!("{}/v1/models", self.config.endpoint); + let mut req = self.http.get(&url).timeout(Duration::from_secs(2)); + if let Some(auth) = self.auth_header() { + req = req.header("Authorization", auth); + } + let ok = req .send() .await .map(|r| r.status().is_success()) .unwrap_or(false); *self.available.lock().unwrap() = Some(ok); if ok { - info!("Ollama reachable at {}", self.config.endpoint); + info!("LLM server reachable at {}", self.config.endpoint); } else { - warn!("Ollama not reachable at {}", self.config.endpoint); + warn!("LLM server not reachable at {}", self.config.endpoint); } ok } - /// Post-process text through Ollama. Returns original text on any failure. + /// Post-process text through the configured LLM server. Returns original text on any failure. pub async fn process(&self, text: &str) -> String { if !self.config.enabled { return text.to_string(); @@ -112,37 +137,46 @@ impl OllamaClient { mode_prompt(&self.config.mode, text) }; - let url = format!("{}/api/generate", self.config.endpoint); - let req = GenerateRequest { + let url = format!("{}/v1/chat/completions", self.config.endpoint); + let req = ChatCompletionRequest { model: &self.config.model, - prompt: &prompt, + messages: vec![ChatMessage { role: "user", content: &prompt }], stream: false, }; - match self.http.post(&url).json(&req).send().await { + let mut builder = self.http.post(&url).json(&req); + if let Some(auth) = self.auth_header() { + builder = builder.header("Authorization", auth); + } + + match builder.send().await { Ok(resp) if resp.status().is_success() => { - match resp.json::().await { - Ok(body) => { - let result = body.response.trim().to_string(); + match resp.json::().await { + Ok(mut body) => { + let result = body + .choices + .pop() + .map(|c| c.message.content.trim().to_string()) + .unwrap_or_else(|| text.to_string()); debug!( input_len = text.len(), output_len = result.len(), - "Ollama processed" + "LLM processed" ); result } Err(e) => { - warn!("Ollama response parse error: {e}"); + warn!("LLM response parse error: {e}"); text.to_string() } } } Ok(resp) => { - warn!("Ollama HTTP {}", resp.status()); + warn!("LLM HTTP {}", resp.status()); text.to_string() } Err(e) => { - warn!("Ollama request error: {e}"); + warn!("LLM request error: {e}"); text.to_string() } } @@ -153,17 +187,21 @@ impl OllamaClient { *self.available.lock().unwrap() = None; } - /// Retrieve the list of available local models from the Ollama instance. + /// Retrieve the list of available models from the OpenAI-API-compatible server. pub async fn list_models(&self) -> Result, String> { - let url = format!("{}/api/tags", self.config.endpoint); - let resp = self.http.get(&url).send().await.map_err(|e| e.to_string())?; + let url = format!("{}/v1/models", self.config.endpoint); + let mut builder = self.http.get(&url); + if let Some(auth) = self.auth_header() { + builder = builder.header("Authorization", auth); + } + let resp = builder.send().await.map_err(|e| e.to_string())?; if resp.status().is_success() { #[derive(serde::Deserialize)] - struct ModelItem { name: String } + struct ModelItem { id: String } #[derive(serde::Deserialize)] - struct TagsResponse { models: Vec } - let tags = resp.json::().await.map_err(|e| e.to_string())?; - Ok(tags.models.into_iter().map(|m| m.name).collect()) + struct ModelsResponse { data: Vec } + let models = resp.json::().await.map_err(|e| e.to_string())?; + Ok(models.data.into_iter().map(|m| m.id).collect()) } else { Err(format!("HTTP error: {}", resp.status())) } diff --git a/docs/api.md b/docs/api.md index 874a0d6..40ae425 100644 --- a/docs/api.md +++ b/docs/api.md @@ -243,15 +243,16 @@ interface AudioDeviceInfo { --- -### Ollama +### OpenAI API -#### `test_ollama(endpoint: string, timeoutSecs: number) → OllamaTestResult` -Pings an Ollama endpoint and lists available models. +#### `test_ollama(endpoint: string, timeoutSecs: number, apiKey?: string) → OllamaTestResult` +Pings an OpenAI-API-compatible server (Ollama, LM Studio, OpenAI, etc.) and lists available models. ```typescript const result = await invoke('test_ollama', { endpoint: 'http://localhost:11434', - timeoutSecs: 5 + timeoutSecs: 5, + apiKey: null }); ``` @@ -390,6 +391,7 @@ interface OllamaConfig { mode: "clean" | "formal" | "casual" | "bullet" | "concise" | "custom"; custom_prompt: string | null; endpoint: string; + api_key: string | null; timeout_secs: number; } diff --git a/docs/configuration.md b/docs/configuration.md index 4585c6f..60aa1b7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -61,6 +61,7 @@ Full schema with defaults: "ollama": { "enabled": false, "endpoint": "http://localhost:11434", + "api_key": null, "model": "llama3.2:1b", "mode": "clean", "custom_prompt": null, @@ -179,10 +180,15 @@ Example with snippets: ### `ollama` section +Despite the key name (kept for config-file backward compatibility), this section configures a connection to +**any OpenAI-API-compatible server** — Ollama, LM Studio, vLLM, OpenAI itself, etc. VoxCtrl talks to the +server's `/v1/models` and `/v1/chat/completions` endpoints. + | Key | Type | Default | Description | |---|---|---|---| -| `enabled` | bool | `false` | Enable Ollama post-processing | -| `endpoint` | string | `"http://localhost:11434"` | Ollama API base URL | +| `enabled` | bool | `false` | Enable LLM post-processing | +| `endpoint` | string | `"http://localhost:11434"` | Base URL of the OpenAI-API-compatible server | +| `api_key` | string or null | `null` | Optional bearer token, sent as `Authorization: Bearer `. Required by most non-localhost servers (e.g. OpenAI). | | `model` | string | `"llama3.2:1b"` | Model name | | `mode` | string | `"clean"` | Rewrite style: `clean`/`formal`/`casual`/`bullet`/`concise`/`custom` | | `custom_prompt` | string or null | `null` | Instruction when mode is `custom`; use `{text}` as placeholder, or text is appended after the prompt | diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 6049b02..892cb5d 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -548,6 +548,7 @@ pub struct OllamaTestResult { pub async fn test_ollama( endpoint: String, timeout_secs: u64, + api_key: Option, ) -> Result { use voxctrl_config::{OllamaConfig, OllamaMode}; let cfg = OllamaConfig { @@ -556,6 +557,7 @@ pub async fn test_ollama( model: String::new(), mode: OllamaMode::Clean, custom_prompt: None, + api_key, timeout_secs, }; let client = voxctrl_llm::OllamaClient::new(cfg); @@ -564,7 +566,7 @@ pub async fn test_ollama( Ok(models) => { Ok(OllamaTestResult { success: true, - message: "Successfully connected to Ollama!".to_string(), + message: "Successfully connected!".to_string(), models, }) } @@ -579,7 +581,7 @@ pub async fn test_ollama( } else { Ok(OllamaTestResult { success: false, - message: format!("Failed to connect to Ollama at '{}'. Make sure Ollama is running.", endpoint), + message: format!("Failed to connect to server at '{}'. Check the URL and API key.", endpoint), models: Vec::new(), }) } diff --git a/src/lib/Settings/OllamaTab.svelte b/src/lib/Settings/OllamaTab.svelte index b6b8b21..0b64674 100644 --- a/src/lib/Settings/OllamaTab.svelte +++ b/src/lib/Settings/OllamaTab.svelte @@ -41,6 +41,7 @@ const res = await invoke("test_ollama", { endpoint: cfg.ollama.endpoint, timeoutSecs: cfg.ollama.timeout_secs, + apiKey: cfg.ollama.api_key, }); testStatus = { success: res.success, message: res.message }; if (res.success) { @@ -65,13 +66,17 @@
-

Ollama LLM Post-Processing

+

OpenAI API LLM Post-Processing

Connection

+ @@ -469,6 +470,65 @@
{/if} + {#if editingTarget.delivery === "openai_api"} +
+
OpenAI API Call Settings
+

Uses the server URL, API key, and default model configured in the OpenAI API settings tab unless overridden below.

+ +
+
+ Instruction / System Prompt +
+ +

Sent as the system message; the transcribed text is sent as the user message.

+
+ +
+
+ Model Override + Optional +
+ +
+ +
+
+ Max Response Tokens + Optional +
+ +
+ +
+
+ Request Timeout (seconds) + Optional +
+ +
+
+ {/if} +
Post-Processing & Output Tuning
diff --git a/src/lib/Settings/TargetsTab.svelte b/src/lib/Settings/TargetsTab.svelte index 7abfe6e..e19022e 100644 --- a/src/lib/Settings/TargetsTab.svelte +++ b/src/lib/Settings/TargetsTab.svelte @@ -170,6 +170,9 @@ {#if t.delivery === "speak"}
Speech: Plays transcribed text via TTS
{/if} + {#if t.delivery === "openai_api"} +
OpenAI API: {t.openai_model || "default model"}
+ {/if}
{#if confirmDeleteTargetId === t.id} diff --git a/src/lib/Settings/routing-types.ts b/src/lib/Settings/routing-types.ts index aa6a131..a29a4b0 100644 --- a/src/lib/Settings/routing-types.ts +++ b/src/lib/Settings/routing-types.ts @@ -33,6 +33,10 @@ export interface OutputTarget { mcp_path?: string; mcp_tool?: string; mcp_args?: Record; + openai_prompt?: string; + openai_model?: string; + openai_max_tokens?: number; + openai_timeout_secs?: number; send_on_release: boolean; append_newline: boolean; strip_newlines: boolean; From f373e6099dbb7ef3c7e4c3583f6f5b49fa6779c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 21:02:42 +0000 Subject: [PATCH 4/7] fix: correct config field access in OpenAI API callback Config is wrapped (state.config.lock().await.data.*), not flat; the callback registered for the new OpenAI API target referenced .ollama directly instead of .data.ollama, breaking the build. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01PThnjsJtj5TRZNBzCLECoi --- src-tauri/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index a3cba6f..c2942d2 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -536,7 +536,7 @@ pub fn run() { voxctrl_routing::targets::set_openai_callback(std::sync::Arc::new(move |req| { let state = state.clone(); Box::pin(async move { - let cfg = state.config.lock().await.ollama.clone(); + let cfg = state.config.lock().await.data.ollama.clone(); let client = voxctrl_llm::OllamaClient::new(cfg); client .complete( From 845c4fdf11c2f7290d0cb9d4ae7deb75edaf2207 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 23 Jun 2026 21:06:12 +0000 Subject: [PATCH 5/7] fix: add missing OpenaiApi fields to test fixtures in src-tauri/lib.rs cargo test failed to compile voxctrl-app because two OutputTarget struct literals in the sequential-delivery test predated the new openai_* fields. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01PThnjsJtj5TRZNBzCLECoi --- src-tauri/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index c2942d2..5ad98d6 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1321,6 +1321,10 @@ mod tests { mcp_path: None, mcp_tool: None, mcp_args: None, + openai_prompt: None, + openai_model: None, + openai_max_tokens: None, + openai_timeout_secs: None, send_on_release: true, append_newline: false, initial_prompt: None, @@ -1353,6 +1357,10 @@ mod tests { mcp_path: None, mcp_tool: None, mcp_args: None, + openai_prompt: None, + openai_model: None, + openai_max_tokens: None, + openai_timeout_secs: None, send_on_release: true, append_newline: false, initial_prompt: None, From 52f2388f06252c855c977e2e29bc77a6c03682c4 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 02:31:38 +0000 Subject: [PATCH 6/7] feat: maintain per-target conversation history for OpenAI API targets Each openai_api target now keeps an in-memory chat log (last 20 messages / 10 exchanges) and replays it with every request, so follow-up dictations like "what did I just ask you?" resolve correctly instead of being treated as an isolated one-shot prompt. History lives only in the running process and resets on hot-reload or restart. Co-Authored-By: Claude Sonnet 4.6 Claude-Session: https://claude.ai/code/session_01PThnjsJtj5TRZNBzCLECoi --- crates/voxctrl-llm/src/lib.rs | 12 ++++-- crates/voxctrl-routing/src/targets.rs | 46 +++++++++++++++++++---- docs/routing.md | 2 + src-tauri/src/lib.rs | 6 +++ src/lib/Settings/TargetEditorModal.svelte | 2 +- 5 files changed, 57 insertions(+), 11 deletions(-) diff --git a/crates/voxctrl-llm/src/lib.rs b/crates/voxctrl-llm/src/lib.rs index 51dd735..0810677 100644 --- a/crates/voxctrl-llm/src/lib.rs +++ b/crates/voxctrl-llm/src/lib.rs @@ -185,12 +185,15 @@ impl OllamaClient { } } - /// Run a one-off chat completion with an arbitrary system/user prompt pair, - /// bypassing the `mode`-based rewrite templates. Used by the OpenAI API - /// output target, which lets each target supply its own instruction. + /// Run a chat completion with an arbitrary system prompt, prior + /// conversation history, and a new user message, bypassing the + /// `mode`-based rewrite templates. Used by the OpenAI API output + /// target, which lets each target supply its own instruction and + /// maintain its own running conversation. pub async fn complete( &self, system_prompt: Option<&str>, + history: &[(String, String)], user_text: &str, model_override: Option<&str>, max_tokens: Option, @@ -203,6 +206,9 @@ impl OllamaClient { messages.push(ChatMessage { role: "system", content: sp }); } } + for (role, content) in history { + messages.push(ChatMessage { role, content }); + } messages.push(ChatMessage { role: "user", content: user_text }); let url = format!("{}/v1/chat/completions", self.config.endpoint); diff --git a/crates/voxctrl-routing/src/targets.rs b/crates/voxctrl-routing/src/targets.rs index 75b9a48..74d7192 100644 --- a/crates/voxctrl-routing/src/targets.rs +++ b/crates/voxctrl-routing/src/targets.rs @@ -16,15 +16,28 @@ pub fn set_speak_callback(callback: SpeakCallback) { let _ = SPEAK_CALLBACK.set(callback); } +/// One prior turn of a target's running conversation, sent back to the +/// server so multi-turn context (e.g. "what did I just ask you") works. +#[derive(Clone)] +pub struct ChatTurn { + pub role: String, + pub content: String, +} + /// Parameters for an OpenAI API output target's chat completion request. pub struct OpenAiCallRequest { pub system_prompt: Option, + pub history: Vec, pub text: String, pub model: Option, pub max_tokens: Option, pub timeout_secs: Option, } +/// Number of messages (user + assistant turns combined) kept per target +/// before older ones are dropped from the running conversation. +const MAX_HISTORY_MESSAGES: usize = 20; + pub type OpenAiCallback = Arc< dyn Fn(OpenAiCallRequest) -> std::pin::Pin> + Send>> + Send @@ -71,7 +84,10 @@ pub fn build_target(config: OutputTarget) -> Box { DeliveryType::Webhook => Box::new(WebhookTarget(config)), DeliveryType::Mcp => Box::new(McpTarget(config)), DeliveryType::Speak => Box::new(SpeakTarget(config)), - DeliveryType::OpenaiApi => Box::new(OpenaiApiTarget(config)), + DeliveryType::OpenaiApi => Box::new(OpenaiApiTarget { + config, + history: tokio::sync::Mutex::new(Vec::new()), + }), } } @@ -818,7 +834,12 @@ impl DeliveryTarget for SpeakTarget { // ── OpenaiApiTarget ─────────────────────────────────────────────────────────── -pub struct OpenaiApiTarget(OutputTarget); +pub struct OpenaiApiTarget { + config: OutputTarget, + /// Running conversation for this target, kept in-memory for the + /// lifetime of the built target (until targets are hot-reloaded). + history: tokio::sync::Mutex>, +} #[async_trait::async_trait] impl DeliveryTarget for OpenaiApiTarget { @@ -826,15 +847,26 @@ impl DeliveryTarget for OpenaiApiTarget { let Some(callback) = OPENAI_CALLBACK.get() else { return DeliveryResult::err("OpenAI API integration not initialized"); }; + let history = self.history.lock().await.clone(); let req = OpenAiCallRequest { - system_prompt: self.0.openai_prompt.clone(), + system_prompt: self.config.openai_prompt.clone(), + history, text: text.to_string(), - model: self.0.openai_model.clone(), - max_tokens: self.0.openai_max_tokens, - timeout_secs: self.0.openai_timeout_secs, + model: self.config.openai_model.clone(), + max_tokens: self.config.openai_max_tokens, + timeout_secs: self.config.openai_timeout_secs, }; match callback(req).await { - Ok(result) => DeliveryResult::ok(result), + Ok(result) => { + let mut hist = self.history.lock().await; + hist.push(ChatTurn { role: "user".into(), content: text.to_string() }); + hist.push(ChatTurn { role: "assistant".into(), content: result.clone() }); + if hist.len() > MAX_HISTORY_MESSAGES { + let excess = hist.len() - MAX_HISTORY_MESSAGES; + hist.drain(0..excess); + } + DeliveryResult::ok(result) + } Err(e) => DeliveryResult::err(e), } } diff --git a/docs/routing.md b/docs/routing.md index 1de8fe7..d603fd6 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -234,6 +234,8 @@ openai_timeout_secs = 30 # Optional: overrides the global request timeo | `openai_max_tokens` | integer | null (server default) | Caps the response length | | `openai_timeout_secs` | integer | null (inherit) | Overrides `ollama.timeout_secs` for this target | +Each `openai_api` target keeps its own running conversation (up to the last 20 messages, i.e. 10 user/assistant exchanges) in memory and replays it with every request, so follow-up dictations like "what did I just ask you?" resolve correctly. This history lives only in the running app process — it is not persisted to disk, and resets whenever targets are hot-reloaded (e.g. after editing targets in Settings) or the app restarts. + --- ### Per-Target Processing diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 5ad98d6..235ae09 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -538,9 +538,15 @@ pub fn run() { Box::pin(async move { let cfg = state.config.lock().await.data.ollama.clone(); let client = voxctrl_llm::OllamaClient::new(cfg); + let history: Vec<(String, String)> = req + .history + .into_iter() + .map(|t| (t.role, t.content)) + .collect(); client .complete( req.system_prompt.as_deref(), + &history, &req.text, req.model.as_deref(), req.max_tokens, diff --git a/src/lib/Settings/TargetEditorModal.svelte b/src/lib/Settings/TargetEditorModal.svelte index 09cdf9e..dcb6d15 100644 --- a/src/lib/Settings/TargetEditorModal.svelte +++ b/src/lib/Settings/TargetEditorModal.svelte @@ -473,7 +473,7 @@ {#if editingTarget.delivery === "openai_api"}
OpenAI API Call Settings
-

Uses the server URL, API key, and default model configured in the OpenAI API settings tab unless overridden below.

+

Uses the server URL, API key, and default model configured in the OpenAI API settings tab unless overridden below. This target remembers the last 10 exchanges so follow-up questions like "what did I just ask you?" work.

From 234e4820ac62889f3aaa1f64571edefcc96c94f2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 24 Jun 2026 02:54:37 +0000 Subject: [PATCH 7/7] Add conversation history to per-binding ollama_enabled pre-processing The earlier openai_api target history fix didn't help users whose Q&A flow goes through the legacy ollama_enabled pre-processing pipeline (OllamaClient::process), which was completely stateless. Add a process_with_history method and a per-binding in-memory history map on InferenceEngine so follow-up questions like "what did I just ask you" resolve correctly for that pipeline too. --- crates/voxctrl-inference/src/lib.rs | 39 +++++++++++++-- crates/voxctrl-llm/src/lib.rs | 76 +++++++++++++++++++++++++++++ docs/routing.md | 2 + 3 files changed, 113 insertions(+), 4 deletions(-) diff --git a/crates/voxctrl-inference/src/lib.rs b/crates/voxctrl-inference/src/lib.rs index 4e32c9b..d0fb22d 100644 --- a/crates/voxctrl-inference/src/lib.rs +++ b/crates/voxctrl-inference/src/lib.rs @@ -43,15 +43,25 @@ pub struct InferenceOutput { // ── Engine ──────────────────────────────────────────────────────────────────── +/// Cap on stored (role, content) pairs per binding — 10 user/assistant exchanges. +const MAX_OLLAMA_HISTORY_MESSAGES: usize = 20; + pub struct InferenceEngine { config: Arc, backend: Box, + /// Running conversation per hotkey binding, for bindings with + /// `ollama_enabled` and a Q&A-style custom prompt. Keyed by binding id. + ollama_history: std::cell::RefCell>>, } impl InferenceEngine { pub fn new(config: Arc) -> Self { let backend = build_backend(&config); - Self { config, backend } + Self { + config, + backend, + ollama_history: std::cell::RefCell::new(std::collections::HashMap::new()), + } } /// Load the selected backend model. Blocks until ready. @@ -211,22 +221,43 @@ impl InferenceEngine { } let client = voxctrl_llm::OllamaClient::new(ollama_cfg); + let history_key = req.binding_id.clone().unwrap_or_default(); + let history = self + .ollama_history + .borrow() + .get(&history_key) + .cloned() + .unwrap_or_default(); + let input_text = processed.clone(); let processed_res = match tokio::runtime::Handle::try_current() { Ok(handle) => { let c = client.clone(); - let text = processed.clone(); + let text = input_text.clone(); + let hist = history.clone(); std::thread::spawn(move || { - handle.block_on(async { c.process(&text).await }) + handle.block_on(async { c.process_with_history(&text, &hist).await }) }).join().unwrap_or(processed) } Err(_) => { if let Ok(rt) = tokio::runtime::Builder::new_current_thread().enable_all().build() { - rt.block_on(async { client.process(&processed).await }) + rt.block_on(async { client.process_with_history(&processed, &history).await }) } else { processed } } }; + + if !history_key.is_empty() { + let mut map = self.ollama_history.borrow_mut(); + let entries = map.entry(history_key).or_default(); + entries.push(("user".to_string(), input_text)); + entries.push(("assistant".to_string(), processed_res.clone())); + let len = entries.len(); + if len > MAX_OLLAMA_HISTORY_MESSAGES { + entries.drain(0..len - MAX_OLLAMA_HISTORY_MESSAGES); + } + } + processed = processed_res; } diff --git a/crates/voxctrl-llm/src/lib.rs b/crates/voxctrl-llm/src/lib.rs index 0810677..d56aff8 100644 --- a/crates/voxctrl-llm/src/lib.rs +++ b/crates/voxctrl-llm/src/lib.rs @@ -185,6 +185,82 @@ impl OllamaClient { } } + /// Like `process`, but replays prior conversation turns before the new + /// templated prompt so hotkey bindings with `ollama_enabled` (e.g. Q&A + /// style custom prompts) can resolve follow-up questions like "what did + /// I just ask you". `history` holds `(role, content)` pairs in order. + pub async fn process_with_history(&self, text: &str, history: &[(String, String)]) -> String { + if !self.config.enabled { + return text.to_string(); + } + if !self.is_available().await { + return text.to_string(); + } + + let prompt = if self.config.mode == OllamaMode::Custom { + if let Some(tmpl) = &self.config.custom_prompt { + if tmpl.contains("{text}") { + tmpl.replace("{text}", text) + } else { + format!("{tmpl}\n\n{text}") + } + } else { + return text.to_string(); + } + } else { + mode_prompt(&self.config.mode, text) + }; + + let url = format!("{}/v1/chat/completions", self.config.endpoint); + let mut messages: Vec = Vec::new(); + for (role, content) in history { + messages.push(ChatMessage { role, content }); + } + messages.push(ChatMessage { role: "user", content: &prompt }); + + let req = ChatCompletionRequest { + model: &self.config.model, + messages, + stream: false, + max_tokens: None, + }; + + let mut builder = self.http.post(&url).json(&req); + if let Some(auth) = self.auth_header() { + builder = builder.header("Authorization", auth); + } + + match builder.send().await { + Ok(resp) if resp.status().is_success() => match resp.json::().await { + Ok(mut body) => { + let result = body + .choices + .pop() + .map(|c| c.message.content.trim().to_string()) + .unwrap_or_else(|| text.to_string()); + debug!( + input_len = text.len(), + output_len = result.len(), + "LLM processed (with history)" + ); + result + } + Err(e) => { + warn!("LLM response parse error: {e}"); + text.to_string() + } + }, + Ok(resp) => { + warn!("LLM HTTP {}", resp.status()); + text.to_string() + } + Err(e) => { + warn!("LLM request error: {e}"); + text.to_string() + } + } + } + /// Run a chat completion with an arbitrary system prompt, prior /// conversation history, and a new user message, bypassing the /// `mode`-based rewrite templates. Used by the OpenAI API output diff --git a/docs/routing.md b/docs/routing.md index d603fd6..619258f 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -261,6 +261,8 @@ quiet_mode = false > [!NOTE] > Ollama post-processing parameters (`ollama_enabled`, `ollama_model`, `ollama_mode`, `ollama_prompt`) are defined per-hotkey binding in `bindings.toml` (or via the Hotkeys UI tab) instead of per-target. This allows you to apply different LLM rewriting styles (e.g. formal, bullet-points) using different hotkeys targeting the same destination. +> +> When `ollama_enabled` is used with a Q&A-style custom prompt (e.g. "answer questions concisely"), each binding keeps its own running conversation (up to the last 10 exchanges) so follow-ups like "what did I just ask you" resolve correctly. This history lives only in the running inference worker — it resets on app restart and is keyed by binding id, so different bindings never share context. ---