path: root/src/dle.rs

//! Reader for the Diccionario de la lengua española (dle.rae.es).
//!
//! Each word page is laid out like this:
//!
//! ```text
//! <article>
//!   <h1 class="c-page-header__title">hablar</h1>
//!   <div class="n2 c-text-intro">Del lat. fabulāri.</div>   ← etymology
//!   <ol class="c-definitions"> <li class="j"> … </li> … </ol> ← senses
//!   <h3>ni hablar</h3> <ol class="c-definitions"> … </ol>     ← set phrases
//! ```
//!
//! Inside each sense, synonyms live in a nested list, so cutting uses
//! [`html::balanced`] instead of looking for the next `</li>`.

use crate::html::{attr, balanced, collapse_ws, find_open, open_tag, strip_tags};
use crate::model::{Abbr, Entry, Locution, Sense};

const CLASS_DEFS: &str = r#"class="c-definitions""#;

/// `true` if the DLE answered with its "word not found" page. It returns 200
/// just like a hit, so the text has to be checked.
pub fn is_not_found(html: &str) -> bool {
    html.contains("no está en el Diccionario") || html.contains("no est&#xE1; en el Diccionario")
}

/// Extracts the entry from a DLE page. `None` if the word is not there.
pub fn parse(html: &str) -> Option<Entry> {
    if is_not_found(html) {
        return None;
    }
    let art_start = find_open(html, 0, "article", Some("o-main__article"))?;
    // The page never closes `<article>`, so the balanced cut would also take
    // the footer. Cut where the page footer starts.
    let article = balanced(html, art_start, "article");
    let article = match article.find("<footer") {
        Some(f) => &article[..f],
        None => article,
    };

    let word = text_of(article, "h1", Some("c-page-header__title"))?;

    // The conjugation and the final synonym lists are separate sections;
    // the definitions are everything before them.
    let conj_at = article.find(r#"id="conjugacion"#);
    let defs_region = &article[..conj_at.unwrap_or(article.len())];

    let etymology = find_open(defs_region, 0, "div", Some("c-text-intro"))
        .map(|i| strip_tags(balanced(defs_region, i, "div")))
        .filter(|s| !s.is_empty());

    let mut senses = Vec::new();
    let mut locutions = Vec::new();
    let mut cursor = 0usize;
    let mut first = true;
    while let Some(ol_at) = find_open(defs_region, cursor, "ol", Some(CLASS_DEFS)) {
        let ol = balanced(defs_region, ol_at, "ol");
        let parsed = parse_senses(ol);
        if first {
            senses = parsed;
            first = false;
        } else if let Some(phrase) = heading_before(defs_region, cursor, ol_at) {
            locutions.push(Locution {
                phrase,
                senses: parsed,
            });
        }
        cursor = ol_at + ol.len();
    }

    Some(Entry {
        word,
        etymology,
        senses,
        locutions,
        // The conjugation is fetched separately, from Wiktionary.
        conjugation: None,
        source: "dle".into(),
    })
}

/// The `<h3>` that titles a set phrase, searched between the previous block and this one.
fn heading_before(h: &str, from: usize, until: usize) -> Option<String> {
    let region = &h[from..until];
    let mut last = None;
    let mut i = 0;
    while let Some(at) = find_open(region, i, "h3", None) {
        last = Some(strip_tags(balanced(region, at, "h3")));
        i = at + 3;
    }
    last.filter(|s| !s.is_empty())
}

fn text_of(h: &str, tag: &str, class: Option<&str>) -> Option<String> {
    let needle = class.map(|c| format!("class=\"{c}\""));
    let at = find_open(h, 0, tag, needle.as_deref())?;
    let s = strip_tags(balanced(h, at, tag));
    (!s.is_empty()).then_some(s)
}

fn parse_senses(ol: &str) -> Vec<Sense> {
    let mut out = Vec::new();
    let mut i = 0;
    while let Some(at) = find_open(ol, i, "li", None) {
        let li = balanced(ol, at, "li");
        // The dictionary marks the article senses with `class="j"` and the
        // set-phrase senses with `class="m"`. Skipping the whole element
        // avoids walking into the synonym list, which is also made of `<li>`.
        let class = attr(open_tag(ol, at), "class").unwrap_or_default();
        if class.starts_with('j') || class.starts_with('m') {
            if let Some(sense) = parse_sense(li) {
                out.push(sense);
            }
        }
        i = at + li.len();
    }
    out
}

fn parse_sense(li: &str) -> Option<Sense> {
    // The footer holds synonyms and antonyms; the body, the definition.
    let footer_at = find_open(li, 0, "div", Some("c-definitions__item-footer"));
    let body = &li[..footer_at.unwrap_or(li.len())];

    let number = element_text(body, "span", "n_acep")
        .map(|s| s.trim().trim_end_matches('.').to_string())
        .filter(|s| !s.is_empty());

    let examples = all_element_texts(body, "span", "h");
    let grammar = abbreviations(body);

    // The text is the body minus the number and the examples.
    let mut text_src = body.to_string();
    for (tag, class) in [("span", "n_acep"), ("span", "h")] {
        text_src = remove_elements(&text_src, tag, class);
    }
    let text = collapse_ws(&strip_tags(&text_src));
    if text.is_empty() && examples.is_empty() {
        return None;
    }

    let (mut synonyms, mut antonyms) = (Vec::new(), Vec::new());
    if let Some(f) = footer_at {
        let footer = balanced(li, f, "div");
        for (label, words) in word_lists(footer) {
            if label.starts_with("Ant") {
                antonyms.extend(words);
            } else {
                synonyms.extend(words);
            }
        }
    }

    Some(Sense {
        number,
        grammar,
        text,
        examples,
        synonyms,
        antonyms,
    })
}

/// The `Sin.:` / `Ant.:` lists in a sense footer, with their label.
fn word_lists(footer: &str) -> Vec<(String, Vec<String>)> {
    let mut out = Vec::new();
    let mut i = 0;
    while let Some(at) = find_open(footer, i, "div", Some(r#"class="c-word-list""#)) {
        let block = balanced(footer, at, "div");
        // The label is told apart by the `title` of its `<abbr>`, which says
        // "Sinónimos o afines" or "Antónimos u opuestos".
        let label = find_open(block, 0, "abbr", Some("sin-header-inline"))
            .and_then(|a| attr(open_tag(block, a), "title"))
            .or_else(|| element_text(block, "div", "c-word-list__label"))
            .unwrap_or_default();
        let words: Vec<String> = all_element_texts(block, "span", "sin")
            .into_iter()
            .map(|w| collapse_ws(w.trim_end_matches([',', '.'])))
            .filter(|w| !w.is_empty())
            .collect();
        if !words.is_empty() {
            out.push((label, words));
        }
        i = at + block.len();
    }
    out
}

/// The abbreviations of a sense with their expanded form (`f.` →
/// «nombre femenino»), which the DLE keeps in the `title` attribute.
fn abbreviations(h: &str) -> Vec<Abbr> {
    let mut out: Vec<Abbr> = Vec::new();
    let mut i = 0;
    while let Some(at) = find_open(h, i, "abbr", None) {
        let el = balanced(h, at, "abbr");
        let short = strip_tags(el);
        if let Some(long) = attr(open_tag(h, at), "title") {
            if !short.is_empty() && !out.iter().any(|a| a.short == short) {
                out.push(Abbr { short, long });
            }
        }
        i = at + el.len();
    }
    out
}

/// Text of the first `<tag class="…class…">`.
fn element_text(h: &str, tag: &str, class: &str) -> Option<String> {
    let needle = format!("class=\"{class}\"");
    let