//! Local voice assistant: it listens, thinks and answers out loud. //! //! It joins three existing engines (Canary to hear, llama.cpp to think and //! qwentts to speak) in a pipeline of threads and channels where no stage //! waits for the next. mod pipeline; mod registry; mod render; mod session; mod supervisor; use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; use anyhow::{bail, Context, Result}; use crossbeam_channel::unbounded; use asist_asr::Recognizer; use asist_audio::{Capture, CaptureBlock, Playback}; use asist_core::config::Config; use asist_core::event::Event; use asist_llm::LlmClient; use asist_tts::TtsClient; use pipeline::Pipeline; use render::Renderer; use session::Session; use supervisor::Supervisor; fn main() -> Result<()> { let args = Args::parse()?; if args.help { println!("{USAGE}"); return Ok(()); } init_logging(&args); let mut config = Config::load(&args.config) .with_context(|| format!("could not load {}", args.config.display()))?; args.apply(&mut config); match args.command { Command::Run => run(config, &args), Command::Devices => devices(), Command::Check => check(&config), } } fn run(config: Config, args: &Args) -> Result<()> { let session = Session::new(); // Clients are created before starting anything: that way whatever is // already listening is detected and no server is started twice. // Shared: the orchestrator converses with it and the camera tool uses it // to describe what it captures. let llm = Arc::new(LlmClient::new(config.llm_authority(), &config.llm)); let tts = TtsClient::new(config.tts_authority(), &config.tts); let (llm_up, tts_up) = (llm.healthy(), tts.healthy()); let mut supervisor = Supervisor::new(&args.log_dir)?; supervisor.start(&config, llm_up, tts_up)?; let timeout = Duration::from_secs(config.supervisor.startup_timeout_secs); eprintln!("Waiting for the engines (up to {} s)…", timeout.as_secs()); llm.wait_ready(timeout).map_err(|e| { anyhow::anyhow!("{e}\nSee {}", supervisor.log_path("llama-server").display()) })?; tts.wait_ready(timeout) .map_err(|e| anyhow::anyhow!("{e}\nSee {}", supervisor.log_path("tts-server").display()))?; llm.warn_if_thinking_template(); if let Some(reference) = &config.tts.reference { tts.register_voice(reference) .context("could not register the cloned voice")?; } // Building the synthesizer graphs is paid here, about 3.5 s that the // first real turn would otherwise eat. if let Err(err) = tts.warmup() { tracing::warn!(target: "tts", %err, "warmup failed; the first turn will be slow"); } eprintln!("Loading the speech-to-text model…"); let recognizer = Recognizer::load(&config.asr)?; let playback = Playback::open(&config.audio)?; let (capture_tx, capture_rx) = unbounded::(); let capture = Capture::open(&config.audio, capture_tx.clone())?; let (tools, skipped) = registry::build(&config, &llm); if tools.is_empty() { eprintln!("Tools: none"); } else { eprintln!("Tools: {}", tools.names().join(", ")); } // Whatever could not be enabled is reported, instead of leaving the user // wondering why the assistant does not search or see. for skip in &skipped { eprintln!(" · {} unavailable: {}", skip.tool, skip.reason); } // The two capabilities that touch something outside the process are // announced: one turns the camera on and the other runs commands. Whoever // starts it should know without having to read the configuration. if tools.get("mirar_la_pantalla").is_some() { eprintln!( "Screen capture ON — at {} px{}", config.screen.width, if config.screen.save_dir.is_empty() { String::new() } else { format!(", saving to {}", config.screen.save_dir) } ); } if tools.get("mirar_por_la_camara").is_some() { eprintln!( "Camera ON — {} at {}x{}{}", config.camera.device.display(), config.camera.width, config.camera.height, if config.camera.save_dir.is_empty() { String::new() } else { format!(", saving frames to {}", config.camera.save_dir) } ); } if config.tools.shell { eprintln!( "Command execution ON — allowed: {}{}", config.tools.shell_allowlist.join(", "), if config.tools.shell_dry_run { " (dry run)" } else { "" } ); } let pipeline = Pipeline::spawn( &config, session.clone(), recognizer, llm, tts, tools, playback.handle(), capture_rx, capture_tx, capture.format, )?; eprintln!( "\nReady. Microphone: {} · Speaker: {} ({} Hz)\nTalk whenever you like. Enter to quit.\n", capture.device_name, playback.device_name, playback.sample_rate ); // Enter quits, but only if someone is there to press it. With redirected // input (under systemd, in a container, with `< /dev/null`) `read_line` // returns EOF immediately and the assistant would quit right after // starting. In that case it waits for a signal. if stdin_is_tty() { let session = session.clone(); std::thread::spawn(move || { let mut line = String::new(); let _ = std::io::stdin().read_line(&mut line); session.request_stop(); }); } else { eprintln!("(non-interactive input: send SIGINT or SIGTERM to quit)"); install_signal_handler(session.clone()); } let mut renderer = Renderer::new(config.asr.partials); let metrics = std::sync::Arc::clone(&pipeline.metrics); let mut next_health_check = std::time::Instant::now() + Duration::from_secs(5); loop { match pipeline.events.recv_timeout(Duration::from_millis(150)) { Ok(event) => { let shutdown = matches!(event, Event::Shutdown); renderer.apply(&event); if shutdown { break; } } Err(crossbeam_channel::RecvTimeoutError::Timeout) => {} Err(crossbeam_channel::RecvTimeoutError::Disconnected) => break, } if session.is_stopping() { break; } // If a server dies mid-session, every following turn would fail with a // transport error without saying why. Better to say it once, with the // log at hand, and quit. if std::time::Instant::now() >= next_health_check { next_health_check = std::time::Instant::now() + Duration::from_secs(5); if let Some((name, code)) = supervisor.crashed() { renderer.finish(); eprintln!( "\n{name} exited unexpectedly (code {}). See {}", code.map(|c| c.to_string()) .unwrap_or_else(|| "unknown".into()), supervisor.log_path(name).display() ); break; } } } renderer.finish(); capture.stop(); pipeline.shutdown(); playback.stop(); supervisor.shutdown(); if metrics.turns() > 0 { eprintln!("\n{}", metrics.summary()); } Ok(()) } /// Lists the audio devices, so they can be named in the configuration. fn devices() -> Result<()> { use asist_audio::describe; use cpal_reexport::traits::HostTrait; let host = cpal_reexport::default_host(); let default_in = host.default_input_device().map(|d| describe(&d)); println!("Inputs:"); for device in host.input_devices()? { let name = describe(&device); let mark = if Some(&name) == default_in.as_ref() { " (default)" } else { "" }; println!(" {name}{mark}"); } let default_out = host.default_output_device().map(|d| describe(&d)); println!("\nOutputs:"); for device in host.output_devices()? { let name = describe(&device); let mark = if Some(&name) == default_out.as_ref() { " (default)" } else { "" }; println!(" {name}{mark}"); } Ok(()) } /// Checks that everything is in place, without opening the microphone. fn check(config: &Config) -> Result<()> { let mut problems = Vec::new(); let ok = |label: &str, detail: String| println!(" ok {label:<22} {detail}"); let paths: [(&str, &std::path::Path); 6] = [ ("asr model", &config.asr.model_dir), ("llama binary", &config.supervisor.llama.binary), ("llm model", &config.supervisor.llama.model), ("chat template", &config.supervisor.llama.chat_template), ("tts binary", &config.supervisor.tts.binary), ("tts model", &config.supervisor.tts.model), ]; println!("Files:"); for (label, path) in paths { if path.exists() { ok(label, path.display().to_string()); } else { println!(" MISSING {label:<20} {}", path.display()); problems.push(format!("missing {label}: {}", path.display())); } } if let Some(reference) = &config.tts.reference { println!("Reference voice «{}»:", reference.name); for (label, path) in [ ("speaker (.spk)", &reference.speaker), ("codes (.rvq)", &reference.codes), ("transcript", &reference.transcript), ] { if path.exists() { ok(label, path.display().to_string()); } else { println!(" MISSING {label:<20} {}", path.display()); problems.push(format!("missing {label}")); } } } println!("Capabilities:"); // Neither is fatal: the assistant converses just the same without them, // so they are reported and not counted as problems. if !config.search.enabled { println!(" - search disabled in the configuration"); } else { match config.search.backend.trim().to_lowercase().as_str() { "tavily" => match std::env::var(&config.search.api_key_env) { Ok(key) if !key.trim().is_empty() => ok( "search (tavily)", format!("key in ${}", config.search.api_key_env), ), _ => println!( " - search (tavily) missing ${}", config.search.api_key_env ), }, "ddgs" | "command" | "comando" => match config.search.command.first() { // A bare name is resolved through PATH; a path must exist, and // better to say so here than in the middle of a question. Some(program) if !program.contains('/') || PathBuf::from(program).exists() => { ok("search (ddgs)", config.search.command.join(" ")) } Some(program) => { println!(" - search (ddgs) {program} does not exist") } None => println!(" - search (ddgs) search.command is empty"), }, "searxng" => ok("search (searxng)", config.search.base_url.clone()), other => println!(" - search unknown backend: «{other}»"), } } if !config.screen.enabled { println!(" - screen disabled in the configuration"); } else { match config.screen.command.first() { Some(program) if !program.contains('/') || PathBuf::from(program).exists() => ok( "screen", format!("{} px · {}", config.screen.width, program), ), Some(program) => println!(" - screen {program} does not exist"), None => println!(" - screen screen.command is empty"), } if std::env::var_os("WAYLAND_DISPLAY").is_none() && std::env::var_os("DISPLAY").is_none() { println!(" - graphical session none; the screen cannot be captured"); } } if !config.camera.enabled { println!(" - camera disabled in the configuration"); } else if config.camera.device.exists() { ok( "camera", format!( "{} at {}x{}", config.camera.device.display(), config.camera.width, config.camera.height ), ); if which("ffmpeg").is_none() { println!(" - ffmpeg missing; needed to capture"); } } else { println!( " - camera {} does not exist", config.camera.device.display() ); } println!("Servers:"); let llm = LlmClient::new(config.llm_authority(), &config.llm); let tts = TtsClient::new(config.tts_authority(), &config.tts); for (label, up, authority) in [ ("llama-server", llm.healthy(), config.llm_authority()), ("tts-server", tts.healthy(), config.tts_authority()), ] { println!( " {:<5} {label:<22} {authority}", if up { "ok" } else { "-" } ); } if llm.healthy() { llm.warn_if_thinking_template(); } if tts.healthy() { match tts.voices() { Ok(voices) if voices.contains(&config.tts.voice) => { println!(" ok voice «{}» registered", config.tts.voice); } Ok(voices) => println!( " - voice «{}» not registered yet (available: {})", config.tts.voice, voices.join(", ") ), Err(err) => println!(" - could not list the voices: {err}"), } } if problems.is_empty() { println!("\nAll set."); Ok(()) } else { bail!( "{} problem(s):\n - {}", problems.len(), problems.join("\n - ") ) } } // cpal is only used here to list devices; it comes through asist-audio. use asist_audio::cpal as cpal_reexport; const USAGE: &str = "\ asistente — local voice assistant USAGE: asistente [COMMAND] [OPTIONS] COMMANDS: run Listen and answer (default) check Check files, models and servers, then exit devices List the audio devices OPTIONS: -c, --config Configuration file [config/asistente.toml] --log-dir Directory for the server logs [logs/] --voice Synthesizer voice --no-manage Do not start the servers; assume they are up --no-partials No partial transcription (saves CPU) --barge-in Allow interrupting the assistant by talking over it --shell Enable system command execution -v, --verbose Debug logging (same as RUST_LOG=debug) -h, --help This help VARIABLES: RUST_LOG Log filter, e.g. «info,tts=debug,latency=info» "; #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Command { Run, Check, Devices, } struct Args { command: Command, config: PathBuf, log_dir: PathBuf, voice: Option, no_manage: bool, no_partials: bool, barge_in: bool, shell: bool, verbose: bool, help: bool, } impl Args { fn parse() -> Result { let mut args = Self { command: Command::Run, config: PathBuf::from("config/asistente.toml"), log_dir: PathBuf::from("logs"), voice: None, no_manage: false, no_partials: false, barge_in: false, shell: false, verbose: false, help: false, }; let mut raw = std::env::args().skip(1); while let Some(arg) = raw.next() { match arg.as_str() { "run" => args.command = Command::Run, "check" => args.command = Command::Check, "devices" => args.command = Command::Devices, "-c" | "--config" => { args.config = raw.next().context("--config needs a path")?.into() } "--log-dir" => args.log_dir = raw.next().context("--log-dir needs a path")?.into(), "--voice" => args.voice = Some(raw.next().context("--voice needs a name")?), "--no-manage" => args.no_manage = true, "--no-partials" => args.no_partials = true, "--barge-in" => args.barge_in = true, "--shell" => args.shell = true, "-v" | "--verbose" => args.verbose = true, "-h" | "--help" => args.help = true, other => bail!("unknown option: {other}\n\n{USAGE}"), } } Ok(args) } /// The command line wins over the file. fn apply(&self, config: &mut Config) { if let Some(voice) = &self.voice { config.tts.voice = voice.clone(); } if self.no_manage { config.supervisor.manage = false; } if self.no_partials { config.asr.partials = false; } if self.barge_in { config.vad.barge_in = true; } if self.shell { config.tools.shell = true; } } } /// Looks an executable up in PATH, to warn about what is missing before it /// fails in the middle of a conversation. fn which(program: &str) -> Option { std::env::var_os("PATH").and_then(|path| { std::env::split_paths(&path) .map(|dir| dir.join(program)) .find(|candidate| candidate.is_file()) }) } /// Is there a person on the other side of standard input? fn stdin_is_tty() -> bool { #[cfg(unix)] { unsafe extern "C" { fn isatty(fd: i32) -> i32; } unsafe { isatty(0) == 1 } } #[cfg(not(unix))] true } /// Orderly shutdown on SIGINT/SIGTERM. /// /// It matters more than it seems: killing the process outright leaves the /// child servers holding GPU memory. #[cfg(unix)] fn install_signal_handler(session: Session) { use std::sync::OnceLock; static SESSION: OnceLock = OnceLock::new(); let _ = SESSION.set(session); extern "C" fn on_signal(_: i32) { if let Some(session) = SESSION.get() { session.request_stop(); } } unsafe extern "C" { fn signal(sig: i32, handler: extern "C" fn(i32)) -> usize; } unsafe { signal(2, on_signal); // SIGINT signal(15, on_signal); // SIGTERM } } #[cfg(not(unix))] fn install_signal_handler(_session: Session) {} fn init_logging(args: &Args) { use tracing_subscriber::{fmt, EnvFilter}; // ONNX Runtime logs every graph transformation at INFO level: several // hundred lines that bury the transcription. It is silenced unless // explicitly requested through RUST_LOG. let default = if args.verbose { "debug,ort=warn" } else { "info,ort=warn" }; let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(default)); fmt() .with_env_filter(filter) // The transcription and the answer go to the text terminal; the log // goes to stderr so they can be separated with a redirection. .with_writer(std::io::stderr) .with_target(true) .without_time() .init(); }