1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
|
//! The screen as an image source.
//!
//! It differs from the camera in the only thing that really matters here: **a
//! screen is text**. And with text this model has an ugly failure mode.
//! Measured on a capture with 13 px UI type, asking for specific details:
//!
//! | Width | Time | Hits | What happens when it fails |
//! |-------|-------|--------|----------------------------|
//! | 640 | 2.4 s | 1 of 3 | it makes the content up |
//! | 960 | 4.5 s | 2 of 3 | it mixes what it read with what it assumed |
//! | 1280 | 7.6 s | 3 of 3 | — |
//!
//! At 640 px it did not say «I cannot read it»: it said the error was «no se
//! pudo abrir el archivo involution» and the meeting was «at 10:00». Neither
//! was in the image. Hence the 1280 px default even though it costs three
//! times the camera: for a voice assistant, confidently stating a wrong time
//! is worse than taking four more seconds.
use std::path::PathBuf;
use std::time::Duration;
use asist_core::error::{Error, Result};
use crate::vision::{capture_jpeg, FrameSource};
const TOOL: &str = "mirar_la_pantalla";
#[derive(Debug, Clone)]
pub struct ScreenConfig {
/// Capture program and its arguments. `{width}` and `{output}` are
/// substituted before running. It must write a JPEG to standard output.
pub command: Vec<String>,
/// Width the capture is scaled down to before sending it to the model.
pub width: u32,
/// A specific monitor; empty = everything there is.
pub output: String,
pub timeout: Duration,
/// Directory to leave captures in. `None` = none is saved.
///
/// It weighs more here than for the camera: a screenshot can hold
/// passwords, private messages and open email.
pub save_dir: Option<PathBuf>,
}
impl Default for ScreenConfig {
fn default() -> Self {
Self {
command: vec![
"scripts/capture-screen.sh".into(),
"{width}".into(),
"{output}".into(),
],
width: 1280,
output: String::new(),
timeout: Duration::from_secs(20),
save_dir: None,
}
}
}
pub struct Screen {
config: ScreenConfig,
}
impl Screen {
pub fn new(config: ScreenConfig) -> Self {
Self { config }
}
fn program(&self) -> Result<&String> {
self.config.command.first().ok_or_else(|| Error::Tool {
tool: TOOL.into(),
message: "screen.command is empty".into(),
})
}
}
impl FrameSource for Screen {
fn label(&self) -> &str {
"pantalla"
}
fn available(&self) -> Result<()> {
let program = self.program()?;
// A bare name is resolved through PATH; a path must exist.
if program.contains('/') && !std::path::Path::new(program).exists() {
return Err(Error::Tool {
tool: TOOL.into(),
message: format!("no existe {program}"),
});
}
// Without a graphical environment there is nothing to capture, and
// better to say so at startup than in the middle of a question.
if std::env::var_os("WAYLAND_DISPLAY").is_none() && std::env::var_os("DISPLAY").is_none() {
return Err(Error::Tool {
tool: TOOL.into(),
message: "no graphical session (neither WAYLAND_DISPLAY nor DISPLAY)".into(),
});
}
Ok(())
}
fn capture(&self) -> Result<Vec<u8>> {
self.available()?;
let program = self.program()?.clone();
let args: Vec<String> = self.config.command[1..]
.iter()
.map(|arg| {
arg.replace("{width}", &self.config.width.to_string())
.replace("{output}", &self.config.output)
})
.collect();
let frame = capture_jpeg(TOOL, program.as_ref(), &args, self.config.timeout)?;
if let Some(dir) = &self.config.save_dir {
crate::camera::save(dir, &frame, "pantalla");
}
Ok(frame)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn config() -> ScreenConfig {
ScreenConfig {
command: vec!["/bin/echo".into(), "{width}".into(), "{output}".into()],
..Default::default()
}
}
#[test]
fn default_width_is_the_measured_minimum_to_read_text() {
assert_eq!(
ScreenConfig::default().width,
1280,
"below 1280 the model makes up what the screen says"
);
}
#[test]
fn no_capture_is_saved_by_default() {
assert!(
ScreenConfig::default().save_dir.is_none(),
"a screenshot can hold passwords and private messages"
);
}
#[test]
fn missing_script_is_detected_before_registering_the_tool() {
let screen = Screen::new(ScreenConfig {
command: vec!["/no/existe/captura.sh".into()],
..Default::default()
});
assert!(screen
.available()
.unwrap_err()
.to_string()
.contains("no existe"));
}
#[test]
fn empty_command_is_rejected() {
let screen = Screen::new(ScreenConfig {
command: vec![],
..Default::default()
});
assert!(screen.available().is_err());
}
#[test]
fn placeholders_are_substituted_before_running() {
// /bin/echo returns the arguments, so the capture fails for not being
// a JPEG; what is checked is that the message carries the width already
// substituted, not the placeholder.
let mut config = config();
config.width = 1280;
config.output = "eDP-1".into();
let screen = Screen::new(config);
if std::env::var_os("WAYLAND_DISPLAY").is_none() && std::env::var_os("DISPLAY").is_none() {
return; // without a graphical session there is nothing to test here
}
let err = screen.capture().unwrap_err().to_string();
assert!(err.contains("no devolvió un JPEG"), "{err}");
}
}
|
