//! What looking through the camera and looking at the screen have in common. //! //! Both tools do the same three steps (get a JPEG, ask the model about it, //! return text) and only differ in where the pixels come from. That is what //! `FrameSource` abstracts. //! //! What they do **not** share is the resolution, and not by oversight: a //! camera scene is understood at 640 px, but at that scale the model does not //! misread UI text, it **makes it up** (see docs/RENDIMIENTO.md). That is why //! each source brings its own. use std::sync::Arc; use std::time::Instant; use asist_core::error::{Error, Result}; use asist_core::http::Cancel; use asist_core::proc; use asist_llm::LlmClient; /// Where the pixels come from. pub trait FrameSource: Send + Sync { /// For error messages and logs: «cámara», «pantalla». fn label(&self) -> &str; /// Captures one frame as JPEG. fn capture(&self) -> Result>; /// `false` if the device or the capture tool is missing. /// /// Checked before registering: declaring a tool that will always fail is /// worse than not having it, because the model calls it, swallows the error /// and wastes the turn. fn available(&self) -> Result<()>; } /// Runs a program that writes a JPEG to standard output. /// /// It is the capture mechanism of both sources: the camera calls ffmpeg and /// the screen a script that knows about compositors. Having it here keeps /// each one from repeating the deadline control and the header check. pub fn capture_jpeg( tool: &str, program: &std::path::Path, args: &[String], timeout: std::time::Duration, ) -> Result> { let fail = |message: String| Error::Tool { tool: tool.to_string(), message, }; let output = proc::run(program, args, timeout, None).map_err(|e| fail(e.to_string()))?; if !output.success() || output.stdout.is_empty() { let stderr = output.stderr.to_lowercase(); // The two most common failures, translated into something actionable. let hint = if stderr.contains("permission denied") { ". Comprueba los permisos del dispositivo" } else if stderr.contains("busy") { ". Otra aplicación lo está usando" } else { "" }; return Err(fail(format!( "no se capturó nada{hint}: {}", output.last_error_line() ))); } // JPEG header. Without this, a corrupt capture reaches the model and // comes back as a generic error that does not say where to look. if output.stdout.len() < 4 || output.stdout[..2] != [0xFF, 0xD8] { return Err(fail(format!( "{} no devolvió un JPEG ({} bytes)", program.display(), output.stdout.len() ))); } tracing::debug!( target: "vision", program = %program.display(), kb = output.stdout.len() / 1024, ms = output.took.as_millis(), "captura" ); Ok(output.stdout) } /// The tool that sees: it captures and asks the model. pub struct VisionTool { source: Box, llm: Arc, name: &'static str, description: &'static str, parameter_hint: &'static str, default_question: &'static str, acknowledgement: &'static str, /// Appended to the user's question before sending it with the image. /// /// It is needed because this request goes outside the history and does not /// get the assistant's voice prompt. style: &'static str, } impl VisionTool { pub fn camera(source: Box, llm: Arc) -> Self { Self { source, llm, name: "mirar_por_la_camara", description: "Toma una foto con la cámara del equipo y responde a una pregunta \ sobre lo que se ve. Úsala cuando te pregunten qué ves, qué hay \ delante, de qué color es algo o cuántas cosas hay.", parameter_hint: "La pregunta del usuario tal cual, sin concretarla más de lo que \ él dijo. Si sólo quiere saber qué hay delante, pon: ¿Qué se ve?", default_question: "¿Qué se ve en esta imagen?", acknowledgement: "Voy a mirar.", style: "Responde en una o dos frases cortas en español, en texto plano, \ describiendo sólo lo que se ve de verdad en la imagen. Si no se \ distingue, dilo.", } } pub fn screen(source: Box, llm: Arc) -> Self { Self { source, llm, name: "mirar_la_pantalla", description: "Hace una captura de la pantalla del equipo y responde a una \ pregunta sobre lo que hay en ella. Úsala cuando te pregunten qué \ hay en pantalla, qué dice un error, qué pone en una ventana o qué \ está abierto.", parameter_hint: "La pregunta del usuario tal cual. Si sólo quiere saber qué hay \ en pantalla, pon: ¿Qué se ve en la pantalla?", default_question: "¿Qué se ve en esta captura de pantalla?", acknowledgement: "Miro la pantalla.", // The warning about not making things up is the most important part of // the whole instruction: when the text is small, this model does not say // it cannot read it, it pulls plausible content out of thin air. style: "Responde en una o dos frases cortas en español, en texto plano. Lee sólo \ lo que de verdad pone en la imagen y no completes lo que no se distinga: \ si el texto está borroso o no se lee, dilo en vez de suponerlo.", } } pub fn available(&self) -> Result<()> { self.source.available() } } impl asist_core::tools::Tool for VisionTool { fn name(&self) -> &str { self.name } fn description(&self) -> &str { self.description } fn parameters(&self) -> serde_json::Value { serde_json::json!({ "type": "object", "properties": { "pregunta": { "type": "string", "description": self.parameter_hint } }, "required": ["pregunta"] }) } /// It turns the camera on or photographs whatever is on screen; in both /// cases it should be announced. fn is_side_effecting(&self) -> bool { true } fn acknowledgement(&self) -> Option<&str> { Some(self.acknowledgement) } fn call(&self, args: &serde_json::Value) -> Result { let question = args .get("pregunta") .and_then(serde_json::Value::as_str) .map(str::trim) .filter(|q| !q.is_empty()) .unwrap_or(self.default_question); let frame = self.source.capture()?; let started = Instant::now(); let answer = self.llm.look( &frame, &format!("{question}\n\n{}", self.style), &Cancel::new(), )?; tracing::info!( target: "vision", source = self.source.label(), kb = frame.len() / 1024, ms = started.elapsed().as_millis(), "descrito" ); Ok(answer) } }