90 lines
1.9 KiB
Rust
90 lines
1.9 KiB
Rust
#[derive(Clone)]
|
||
pub struct LlmService;
|
||
|
||
|
||
static LMSTUDIO_URL: LazyLock<Option<String>> = LazyLock::new(|| env::var("LMSTUDIO_URL").ok());
|
||
static LMSTUDIO_MODEL: LazyLock<Option<String>> = LazyLock::new(|| env::var("LMSTUDIO_MODEL").ok());
|
||
|
||
impl LlmService {
|
||
|
||
pub fn new() -> Self {
|
||
Self {}
|
||
}
|
||
|
||
pub fn enabled(&self) -> bool {
|
||
LMSTUDIO_URL.is_some()
|
||
}
|
||
|
||
pub async fn query(&self, message: &ChatMsg) -> ApiResult<ChatMsg> {
|
||
let Some(url) = LMSTUDIO_URL.clone() else {
|
||
return Err(AppError::internal("AI not enabled!"))
|
||
};
|
||
|
||
let model = LMSTUDIO_MODEL.clone().unwrap_or_else(|| "gpt-oss-20b".into());
|
||
|
||
let client = reqwest::Client::new();
|
||
|
||
// Build the request body
|
||
let payload = LlmRequest {
|
||
model, // whatever model you run locally
|
||
messages: vec![Message {
|
||
role: "user".into(),
|
||
content: message.text.clone(),
|
||
}],
|
||
};
|
||
|
||
// POST to lm‑studio (default 127.0.0.1:1234)
|
||
let resp = client
|
||
.post(url)
|
||
.json(&payload)
|
||
.send()
|
||
.await
|
||
.map_err(|_| AppError::internal("Failed to make request to LLM server"))?;
|
||
|
||
// The API returns a JSON with `choices[].message.content`
|
||
#[derive(Deserialize)]
|
||
struct LlmResponse {
|
||
choices: Vec<Choice>,
|
||
}
|
||
#[derive(Deserialize)]
|
||
struct Choice {
|
||
message: Message,
|
||
}
|
||
|
||
let llm_resp: LlmResponse = resp
|
||
.json()
|
||
.await
|
||
.map_err(|_| AppError::internal("Failed to parse LLM response"))?;
|
||
|
||
Ok(ChatMsg {
|
||
display_name: Some(String::from("llm")),
|
||
user_id: 0,
|
||
text: llm_resp.choices[0].message.content.clone(),
|
||
timestamp: chrono::Utc::now(),
|
||
})
|
||
}
|
||
}
|
||
|
||
use std::env;
|
||
use std::sync::LazyLock;
|
||
// src/llm.rs
|
||
use serde::{Deserialize, Serialize};
|
||
|
||
use crate::api::chat::ChatMsg;
|
||
use crate::error::{ApiResult, AppError};
|
||
use crate::svc::chat_svc::ChatService;
|
||
|
||
#[derive(Serialize)]
|
||
struct LlmRequest {
|
||
model: String,
|
||
messages: Vec<Message>,
|
||
}
|
||
|
||
#[derive(Serialize, Deserialize)]
|
||
struct Message {
|
||
role: String, // "user" or "assistant"
|
||
content: String,
|
||
}
|
||
|
||
|