drae: the RAE Spanish dictionary in the terminalHEADmain

1 files changed, 309 insertions, 0 deletions

diff --git a/src/dle.rs b/src/dle.rs
new file mode 100644
index 0000000..ed3d0e5
--- /dev/null
+++ b/src/dle.rs
@@ -0,0 +1,309 @@
+//! Reader for the Diccionario de la lengua española (dle.rae.es).
+//!
+//! Each word page is laid out like this:
+//!
+//! ```text
+//! <article>
+//!   <h1 class="c-page-header__title">hablar</h1>
+//!   <div class="n2 c-text-intro">Del lat. fabulāri.</div>   ← etymology
+//!   <ol class="c-definitions"> <li class="j"> … </li> … </ol> ← senses
+//!   <h3>ni hablar</h3> <ol class="c-definitions"> … </ol>     ← set phrases
+//! ```
+//!
+//! Inside each sense, synonyms live in a nested list, so cutting uses
+//! [`html::balanced`] instead of looking for the next `</li>`.
+
+use crate::html::{attr, balanced, collapse_ws, find_open, open_tag, strip_tags};
+use crate::model::{Abbr, Entry, Locution, Sense};
+
+const CLASS_DEFS: &str = r#"class="c-definitions""#;
+
+/// `true` if the DLE answered with its "word not found" page. It returns 200
+/// just like a hit, so the text has to be checked.
+pub fn is_not_found(html: &str) -> bool {
+    html.contains("no está en el Diccionario") || html.contains("no est&#xE1; en el Diccionario")
+}
+
+/// Extracts the entry from a DLE page. `None` if the word is not there.
+pub fn parse(html: &str) -> Option<Entry> {
+    if is_not_found(html) {
+        return None;
+    }
+    let art_start = find_open(html, 0, "article", Some("o-main__article"))?;
+    // The page never closes `<article>`, so the balanced cut would also take
+    // the footer. Cut where the page footer starts.
+    let article = balanced(html, art_start, "article");
+    let article = match article.find("<footer") {
+        Some(f) => &article[..f],
+        None => article,
+    };
+
+    let word = text_of(article, "h1", Some("c-page-header__title"))?;
+
+    // The conjugation and the final synonym lists are separate sections;
+    // the definitions are everything before them.
+    let conj_at = article.find(r#"id="conjugacion"#);
+    let defs_region = &article[..conj_at.unwrap_or(article.len())];
+
+    let etymology = find_open(defs_region, 0, "div", Some("c-text-intro"))
+        .map(|i| strip_tags(balanced(defs_region, i, "div")))
+        .filter(|s| !s.is_empty());
+
+    let mut senses = Vec::new();
+    let mut locutions = Vec::new();
+    let mut cursor = 0usize;
+    let mut first = true;
+    while let Some(ol_at) = find_open(defs_region, cursor, "ol", Some(CLASS_DEFS)) {
+        let ol = balanced(defs_region, ol_at, "ol");
+        let parsed = parse_senses(ol);
+        if first {
+            senses = parsed;
+            first = false;
+        } else if let Some(phrase) = heading_before(defs_region, cursor, ol_at) {
+            locutions.push(Locution {
+                phrase,
+                senses: parsed,
+            });
+        }
+        cursor = ol_at + ol.len();
+    }
+
+    Some(Entry {
+        word,
+        etymology,
+        senses,
+        locutions,
+        // The conjugation is fetched separately, from Wiktionary.
+        conjugation: None,
+        source: "dle".into(),
+    })
+}
+
+/// The `<h3>` that titles a set phrase, searched between the previous block and this one.
+fn heading_before(h: &str, from: usize, until: usize) -> Option<String> {
+    let region = &h[from..until];
+    let mut last = None;
+    let mut i = 0;
+    while let Some(at) = find_open(region, i, "h3", None) {
+        last = Some(strip_tags(balanced(region, at, "h3")));
+        i = at + 3;
+    }
+    last.filter(|s| !s.is_empty())
+}
+
+fn text_of(h: &str, tag: &str, class: Option<&str>) -> Option<String> {
+    let needle = class.map(|c| format!("class=\"{c}\""));
+    let at = find_open(h, 0, tag, needle.as_deref())?;
+    let s = strip_tags(balanced(h, at, tag));
+    (!s.is_empty()).then_some(s)
+}
+
+fn parse_senses(ol: &str) -> Vec<Sense> {
+    let mut out = Vec::new();
+    let mut i = 0;
+    while let Some(at) = find_open(ol, i, "li", None) {
+        let li = balanced(ol, at, "li");
+        // The dictionary marks the article senses with `class="j"` and the
+        // set-phrase senses with `class="m"`. Skipping the whole element
+        // avoids walking into the synonym list, which is also made of `<li>`.
+        let class = attr(open_tag(ol, at), "class").unwrap_or_default();
+        if class.starts_with('j') || class.starts_with('m') {
+            if let Some(sense) = parse_sense(li) {
+                out.push(sense);
+            }
+        }
+        i = at + li.len();
+    }
+    out
+}
+
+fn parse_sense(li: &str) -> Option<Sense> {
+    // The footer holds synonyms and antonyms; the body, the definition.
+    let footer_at = find_open(li, 0, "div", Some("c-definitions__item-footer"));
+    let body = &li[..footer_at.unwrap_or(li.len())];
+
+    let number = element_text(body, "span", "n_acep")
+        .map(|s| s.trim().trim_end_matches('.').to_string())
+        .filter(|s| !s.is_empty());
+
+    let examples = all_element_texts(body, "span", "h");
+    let grammar = abbreviations(body);
+
+    // The text is the body minus the number and the examples.
+    let mut text_src = body.to_string();
+    for (tag, class) in [("span", "n_acep"), ("span", "h")] {
+        text_src = remove_elements(&text_src, tag, class);
+    }
+    let text = collapse_ws(&strip_tags(&text_src));
+    if text.is_empty() && examples.is_empty() {
+        return None;
+    }
+
+    let (mut synonyms, mut antonyms) = (Vec::new(), Vec::new());
+    if let Some(f) = footer_at {
+        let footer = balanced(li, f, "div");
+        for (label, words) in word_lists(footer) {
+            if label.starts_with("Ant") {
+                antonyms.extend(words);
+            } else {
+                synonyms.extend(words);
+            }
+        }
+    }
+
+    Some(Sense {
+        number,
+        grammar,
+        text,
+        examples,
+        synonyms,
+        antonyms,
+    })
+}
+
+/// The `Sin.:` / `Ant.:` lists in a sense footer, with their label.
+fn word_lists(footer: &str) -> Vec<(String, Vec<String>)> {
+    let mut out = Vec::new();
+    let mut i = 0;
+    while let Some(at) = find_open(footer, i, "div", Some(r#"class="c-word-list""#)) {
+        let block = balanced(footer, at, "div");
+        // The label is told apart by the `title` of its `<abbr>`, which says
+        // "Sinónimos o afines" or "Antónimos u opuestos".
+        let label = find_open(block, 0, "abbr", Some("sin-header-inline"))
+            .and_then(|a| attr(open_tag(block, a), "title"))
+            .or_else(|| element_text(block, "div", "c-word-list__label"))
+            .unwrap_or_default();
+        let words: Vec<String> = all_element_texts(block, "span", "sin")
+            .into_iter()
+            .map(|w| collapse_ws(w.trim_end_matches([',', '.'])))
+            .filter(|w| !w.is_empty())
+            .collect();
+        if !words.is_empty() {
+            out.push((label, words));
+        }
+        i = at + block.len();
+    }
+    out
+}
+
+/// The abbreviations of a sense with their expanded form (`f.` →
+/// «nombre femenino»), which the DLE keeps in the `title` attribute.
+fn abbreviations(h: &str) -> Vec<Abbr> {
+    let mut out: Vec<Abbr> = Vec::new();
+    let mut i = 0;
+    while let Some(at) = find_open(h, i, "abbr", None) {
+        let el = balanced(h, at, "abbr");
+        let short = strip_tags(el);
+        if let Some(long) = attr(open_tag(h, at), "title") {
+            if !short.is_empty() && !out.iter().any(|a| a.short == short) {
+                out.push(Abbr { short, long });
+            }
+        }
+        i = at + el.len();
+    }
+    out
+}
+
+/// Text of the first `<tag class="…class…">`.
+fn element_text(h: &str, tag: &str, class: &str) -> Option<String> {
+    let needle = format!("class=\"{class}\"");
+    let at = find_open(h, 0, tag, Some(&needle))?;
+    Some(strip_tags(balanced(h, at, tag)))
+}
+
+/// Text of every `<tag class="…class…">`, in order.
+fn all_element_texts(h: &str, tag: &str, class: &str) -> Vec<String> {
+    let needle = format!("class=\"{class}\"");
+    let mut out = Vec::new();
+    let mut i = 0;
+    while let Some(at) = find_open(h, i, tag, Some(&needle)) {
+        let el = balanced(h, at, tag);
+        let t = strip_tags(el);
+        if !t.is_empty() {
+            out.push(t);
+        }
+        i = at + el.len();
+    }
+    out
+}
+
+/// Removes the given elements entirely, content included.
+fn remove_elements(h: &str, tag: &str, class: &str) -> String {
+    let needle = format!("class=\"{class}\"");
+    let mut out = String::with_capacity(h.len());
+    let mut i = 0;
+    while let Some(at) = find_open(h, i, tag, Some(&needle)) {
+        out.push_str(&h[i..at]);
+        // Leave a space so neighbouring words do not run together.
+        out.push(' ');
+        i = at + balanced(h, at, tag).len();
+    }
+    out.push_str(&h[i..]);
+    out
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// A real dle.rae.es page, trimmed to four senses and one set phrase.
+    /// The live path almost never answers (Cloudflare challenges anything
+    /// that is not a browser), so this fixture is what keeps the reader
+    /// verified.
+    const CASA: &str = include_str!("../tests/fixtures/dle-casa.html");
+
+    #[test]
+    fn reads_headword_and_etymology() {
+        let e = parse(CASA).expect("should parse the page");
+        assert_eq!(e.word, "casa");
+        assert_eq!(e.etymology.as_deref(), Some("Del lat. casa 'choza'."));
+        assert_eq!(e.source, "dle");
+    }
+
+    #[test]
+    fn reads_senses_with_marks_examples_and_synonyms() {
+        let e = parse(CASA).unwrap();
+        assert_eq!(e.senses.len(), 4, "senses: {}", e.senses.len());
+
+        let first = &e.senses[0];
+        assert_eq!(first.number.as_deref(), Some("1"));
+        // The text keeps the grammar mark and the final dot, as the
+        // dictionary prints it.
+        assert_eq!(first.text, "f. Edificio para habitar.");
+        assert_eq!(first.examples, ["Una casa de ocho plantas."]);
+        assert!(first.synonyms.contains(&"vivienda".to_string()));
+        assert!(first.synonyms.contains(&"casalicio".to_string()));
+        // The example must not leak into the definition.
+        assert!(!first.text.contains("ocho plantas"));
+        // Nor the synonyms.
+        assert!(!first.text.contains("vivienda"));
+    }
+
+    #[test]
+    fn expands_abbreviations_from_the_title_attribute() {
+        let e = parse(CASA).unwrap();
+        let g = &e.senses[0].grammar;
+        assert!(
+            g.iter()
+                .any(|a| a.short == "f." && a.long == "nombre femenino"),
+            "abbreviations: {g:?}"
+        );
+    }
+
+    #[test]
+    fn reads_locutions() {
+        let e = parse(CASA).unwrap();
+        assert_eq!(e.locutions.len(), 1, "set phrases: {:?}", e.locutions);
+        assert!(!e.locutions[0].phrase.is_empty());
+        assert!(!e.locutions[0].senses.is_empty());
+    }
+
+    #[test]
+    fn detects_the_not_found_page() {
+        assert!(is_not_found(
+            "<p>La palabra «asdfgh» no está en el Diccionario</p>"
+        ));
+        assert!(!is_not_found(CASA));
+        assert!(parse("<p>no está en el Diccionario</p>").is_none());
+    }
+}