path: root/crates/asist-tools/src/vision.rs

//! What looking through the camera and looking at the screen have in common.
//!
//! Both tools do the same three steps (get a JPEG, ask the model about it,
//! return text) and only differ in where the pixels come from. That is what
//! `FrameSource` abstracts.
//!
//! What they do **not** share is the resolution, and not by oversight: a
//! camera scene is understood at 640 px, but at that scale the model does not
//! misread UI text, it **makes it up** (see docs/RENDIMIENTO.md). That is why
//! each source brings its own.

use std::sync::Arc;
use std::time::Instant;

use asist_core::error::{Error, Result};
use asist_core::http::Cancel;
use asist_core::proc;
use asist_llm::LlmClient;

/// Where the pixels come from.
pub trait FrameSource: Send + Sync {
    /// For error messages and logs: «cámara», «pantalla».
    fn label(&self) -> &str;

    /// Captures one frame as JPEG.
    fn capture(&self) -> Result<Vec<u8>>;

    /// `false` if the device or the capture tool is missing.
    ///
    /// Checked before registering: declaring a tool that will always fail is
    /// worse than not having it, because the model calls it, swallows the error
    /// and wastes the turn.
    fn available(&self) -> Result<()>;
}

/// Runs a program that writes a JPEG to standard output.
///
/// It is the capture mechanism of both sources: the camera calls ffmpeg and
/// the screen a script that knows about compositors. Having it here keeps
/// each one from repeating the deadline control and the header check.
pub fn capture_jpeg(
    tool: &str,
    program: &std::path::Path,
    args: &[String],
    timeout: std::time::Duration,
) -> Result<Vec<u8>> {
    let fail = |message: String| Error::Tool {
        tool: tool.to_string(),
        message,
    };

    let output = proc::run(program, args, timeout, None).map_err(|e| fail(e.to_string()))?;

    if !output.success() || output.stdout.is_empty() {
        let stderr = output.stderr.to_lowercase();
        // The two most common failures, translated into something actionable.
        let hint = if stderr.contains("permission denied") {
            ". Comprueba los permisos del dispositivo"
        } else if stderr.contains("busy") {
            ". Otra aplicación lo está usando"
        } else {
            ""
        };
        return Err(fail(format!(
            "no se capturó nada{hint}: {}",
            output.last_error_line()
        )));
    }

    // JPEG header. Without this, a corrupt capture reaches the model and
    // comes back as a generic error that does not say where to look.
    if output.stdout.len() < 4 || output.stdout[..2] != [0xFF, 0xD8] {
        return Err(fail(format!(
            "{} no devolvió un JPEG ({} bytes)",
            program.display(),
            output.stdout.len()
        )));
    }

    tracing::debug!(
        target: "vision",
        program = %program.display(),
        kb = output.stdout.len() / 1024,
        ms = output.took.as_millis(),
        "captura"
    );
    Ok(output.stdout)
}

/// The tool that sees: it captures and asks the model.
pub struct VisionTool {
    source: Box<dyn FrameSource>,
    llm: Arc<LlmClient>,
    name: &'static str,
    description: &'static str,
    parameter_hint: &'static str,
    default_question: &'static str,
    acknowledgement: &'static str,
    /// Appended to the user's question before sending it with the image.
    ///
    /// It is needed because this request goes outside the history and does not
    /// get the assistant's voice prompt.
    style: &'static str,
}

impl VisionTool {
    pub fn camera(source: Box<dyn FrameSource>, llm: Arc<LlmClient>) -> Self {
        Self {
            source,
            llm,
            name: "mirar_por_la_camara",
            description: "Toma una foto con la cámara del equipo y responde a una pregunta \
                          sobre lo que se ve. Úsala cuando te pregunten qué ves, qué hay \
                          delante, de qué color es algo o cuántas cosas hay.",
            parameter_hint: "La pregunta del usuario tal cual, sin concretarla más de lo que \
                             él dijo. Si sólo quiere saber qué hay delante, pon: ¿Qué se ve?",
            default_question: "¿Qué se ve en esta imagen?",
            acknowledgement: "Voy a mirar.",
            style: "Responde en una o dos frases cortas en español, en texto plano, \
                    describiendo sólo lo que se ve de verdad en la imagen. Si no se \
                    distingue, dilo.",
        }
    }

    pub fn screen(source: Box<dyn FrameSource>, llm: Arc<LlmClient>) -> Self {
        Self {
            source,
            llm,
            name: "mirar_la_pantalla",
            description: "Hace una captura de la pantalla del equipo y responde a una \
                          pregunta sobre lo que hay en ella. Úsala cuando te pregunten qué \
                          hay en pantalla, qué dice un error, qué pone en una ventana o qué \
                          está abierto.",
            parameter_hint: "La pregunta del usuario tal cual. Si sólo quiere saber qué hay \
                             en pantalla, pon: ¿Qué se ve en la pantalla?",
            default_question: "¿Qué se ve en esta captura de pantalla?",
            acknowledgement: "Miro la pantalla.",
            // The warning about not making things up is the most important part of
            // the whole instruction: when the text is small, this model does not say
            // it cannot read it, it pulls plausible content out of thin air.
            style: "Responde en una o dos frases cortas en español, en texto plano. Lee sólo \
                    lo que de verdad pone en la imagen y no completes lo que no se distinga: \
                    si el texto está borroso o no se lee, dilo en vez de suponerlo.",
        }
    }

    pub fn available(&self) -> Result<()> {
        self.source.available()
    }
}

impl asist_core::tools::Tool for VisionTool {
    fn name(&self) -> &str {
        self.name
    }

    fn description(&self) -> &str {
        self.description
    }

    fn parameters(&self) -> serde_json::Value {
        serde_json::json!({
            "type": "object",
            "properties": {
                "pregunta": { "type": "string", "description": self.parameter_hint }
            },
            "required": ["pregunta"]
        })
    }

    /// It turns the camera on or photographs whatever is on screen; in both
    /// cases it should be announced.
    fn is_side_effecting(&self) -> bool {
        true
    }

    fn acknowledgement(&self) -> Option<&str> {
        Some(self.acknowledgement)
    }

    fn call(&self, args: &serde_json::Value) -> Result<String> {
        let question = args
            .get("pregunta")
            .and_then(serde_json::Value::as_str)
            .map(str::trim)
            .filter(|q| !q.is_empty())
            .unwrap_or(self.default_question);

        let frame = self.source.capture()?;
        let started = Instant::now();
        let answer = self.llm.look(
            &frame,
            &format!("{question}\n\n{}", self.style),
            &Cancel::new(),
        )?;

        tracing::info!(
            target: "vision",
            source = self.source.label(),
            kb = frame.len() / 1024,
            ms = started.elapsed().as_millis(),
            "descrito"
        );
        Ok(answer)
    }
}