//! The screen as an image source. //! //! It differs from the camera in the only thing that really matters here: **a //! screen is text**. And with text this model has an ugly failure mode. //! Measured on a capture with 13 px UI type, asking for specific details: //! //! | Width | Time | Hits | What happens when it fails | //! |-------|-------|--------|----------------------------| //! | 640 | 2.4 s | 1 of 3 | it makes the content up | //! | 960 | 4.5 s | 2 of 3 | it mixes what it read with what it assumed | //! | 1280 | 7.6 s | 3 of 3 | — | //! //! At 640 px it did not say «I cannot read it»: it said the error was «no se //! pudo abrir el archivo involution» and the meeting was «at 10:00». Neither //! was in the image. Hence the 1280 px default even though it costs three //! times the camera: for a voice assistant, confidently stating a wrong time //! is worse than taking four more seconds. use std::path::PathBuf; use std::time::Duration; use asist_core::error::{Error, Result}; use crate::vision::{capture_jpeg, FrameSource}; const TOOL: &str = "mirar_la_pantalla"; #[derive(Debug, Clone)] pub struct ScreenConfig { /// Capture program and its arguments. `{width}` and `{output}` are /// substituted before running. It must write a JPEG to standard output. pub command: Vec, /// Width the capture is scaled down to before sending it to the model. pub width: u32, /// A specific monitor; empty = everything there is. pub output: String, pub timeout: Duration, /// Directory to leave captures in. `None` = none is saved. /// /// It weighs more here than for the camera: a screenshot can hold /// passwords, private messages and open email. pub save_dir: Option, } impl Default for ScreenConfig { fn default() -> Self { Self { command: vec![ "scripts/capture-screen.sh".into(), "{width}".into(), "{output}".into(), ], width: 1280, output: String::new(), timeout: Duration::from_secs(20), save_dir: None, } } } pub struct Screen { config: ScreenConfig, } impl Screen { pub fn new(config: ScreenConfig) -> Self { Self { config } } fn program(&self) -> Result<&String> { self.config.command.first().ok_or_else(|| Error::Tool { tool: TOOL.into(), message: "screen.command is empty".into(), }) } } impl FrameSource for Screen { fn label(&self) -> &str { "pantalla" } fn available(&self) -> Result<()> { let program = self.program()?; // A bare name is resolved through PATH; a path must exist. if program.contains('/') && !std::path::Path::new(program).exists() { return Err(Error::Tool { tool: TOOL.into(), message: format!("no existe {program}"), }); } // Without a graphical environment there is nothing to capture, and // better to say so at startup than in the middle of a question. if std::env::var_os("WAYLAND_DISPLAY").is_none() && std::env::var_os("DISPLAY").is_none() { return Err(Error::Tool { tool: TOOL.into(), message: "no graphical session (neither WAYLAND_DISPLAY nor DISPLAY)".into(), }); } Ok(()) } fn capture(&self) -> Result> { self.available()?; let program = self.program()?.clone(); let args: Vec = self.config.command[1..] .iter() .map(|arg| { arg.replace("{width}", &self.config.width.to_string()) .replace("{output}", &self.config.output) }) .collect(); let frame = capture_jpeg(TOOL, program.as_ref(), &args, self.config.timeout)?; if let Some(dir) = &self.config.save_dir { crate::camera::save(dir, &frame, "pantalla"); } Ok(frame) } } #[cfg(test)] mod tests { use super::*; fn config() -> ScreenConfig { ScreenConfig { command: vec!["/bin/echo".into(), "{width}".into(), "{output}".into()], ..Default::default() } } #[test] fn default_width_is_the_measured_minimum_to_read_text() { assert_eq!( ScreenConfig::default().width, 1280, "below 1280 the model makes up what the screen says" ); } #[test] fn no_capture_is_saved_by_default() { assert!( ScreenConfig::default().save_dir.is_none(), "a screenshot can hold passwords and private messages" ); } #[test] fn missing_script_is_detected_before_registering_the_tool() { let screen = Screen::new(ScreenConfig { command: vec!["/no/existe/captura.sh".into()], ..Default::default() }); assert!(screen .available() .unwrap_err() .to_string() .contains("no existe")); } #[test] fn empty_command_is_rejected() { let screen = Screen::new(ScreenConfig { command: vec![], ..Default::default() }); assert!(screen.available().is_err()); } #[test] fn placeholders_are_substituted_before_running() { // /bin/echo returns the arguments, so the capture fails for not being // a JPEG; what is checked is that the message carries the width already // substituted, not the placeholder. let mut config = config(); config.width = 1280; config.output = "eDP-1".into(); let screen = Screen::new(config); if std::env::var_os("WAYLAND_DISPLAY").is_none() && std::env::var_os("DISPLAY").is_none() { return; // without a graphical session there is nothing to test here } let err = screen.capture().unwrap_err().to_string(); assert!(err.contains("no devolvió un JPEG"), "{err}"); } }