path: root/src/wiktionary.rs

//! Verb conjugation from the Spanish Wiktionary.
//!
//! The DLE publishes its own tables, but dle.rae.es answers non-browser
//! clients with a Cloudflare challenge, so the conjugation comes from here:
//! an API meant for programmatic use, with CC BY-SA content, which also
//! includes voseo («vos hablás») that the RAE tables leave out.
//!
//! The table is a regular 32-row grid:
//!
//! ```text
//! <th colspan="8">Modo indicativo</th>        ← a mood starts
//! <th></th><th>yo</th><th>tú</th>…            ← pronouns (already in every cell)
//! <th>Presente</th><td>yo hablo</td>…         ← a tense
//! ```

use anyhow::{Context, Result};
use serde::Deserialize;

use crate::client::{percent_encode, Http, CLIENT_UA};
use crate::html::{balanced, find_open, strip_tags};
use crate::model::{ConjGroup, Conjugation, Tense};

const API: &str = "https://es.wiktionary.org/w/api.php";

#[derive(Deserialize)]
struct Response {
    parse: Option<Parse>,
}

#[derive(Deserialize)]
struct Parse {
    title: String,
    text: String,
}

/// Downloads and builds the conjugation of a verb. `Ok(None)` if the page
/// does not exist or has no conjugation table (because it is not a verb).
pub fn conjugation(http: &Http, verb: &str) -> Result<Option<Conjugation>> {
    let url = format!(
        "{API}?action=parse&page={}&prop=text&format=json&formatversion=2",
        percent_encode(verb)
    );
    let (status, body) = http.get(&url, CLIENT_UA)?;
    if status != 200 {
        anyhow::bail!("es.wiktionary.org respondió {status}");
    }
    let res: Response =
        serde_json::from_str(&body).context("el Wikcionario devolvió un JSON inesperado")?;
    // Without `parse` the page does not exist; the error comes in another field.
    let Some(parse) = res.parse else {
        return Ok(None);
    };
    // Only the Spanish section: Wiktionary puts every language that shares
    // the spelling on one page, and several have their own table. «casa»,
    // for one, is a verb in Romanian.
    let section = spanish_section(&parse.text).unwrap_or(&parse.text);
    Ok(parse_table(section).map(|groups| Conjugation {
        title: format!("Conjugación de «{}»", parse.title),
        groups,
    }))
}

/// Cuts out the «Español» section, from its heading to the next language's.
fn spanish_section(html: &str) -> Option<&str> {
    let mut start: Option<usize> = None;
    let mut i = 0;
    while let Some(at) = find_open(html, i, "h2", None) {
        let el = balanced(html, at, "h2");
        let title = strip_tags(el);
        i = at + el.len();
        match start {
            Some(s) => return Some(&html[s..at]),
            None if title == "Español" => start = Some(at),
            None => {}
        }
    }
    start.map(|s| &html[s..])
}

/// Finds the conjugation table and turns it into moods and tenses.
fn parse_table(html: &str) -> Option<Vec<ConjGroup>> {
    let table = find_conjugation_table(html)?;
    let mut groups: Vec<ConjGroup> = Vec::new();
    let mut i = 0;

    while let Some(at) = find_open(table, i, "tr", None) {
        let tr = balanced(table, at, "tr");
        i = at + tr.len();

        let (headers, data) = (cells(tr, "th"), cells(tr, "td"));

        // A row with a single wide header starts a mood.
        if data.is_empty() && headers.len() == 1 {
            let title = headers[0].clone();
            if is_divider(&title) {
                continue;
            }
            groups.push(ConjGroup {
                title,
                tenses: vec![],
            });
            continue;
        }

        // The pronoun row adds nothing: every cell already repeats them.
        if data.is_empty() {
            continue;
        }

        let Some(group) = groups.last_mut() else {
            continue;
        };
        let Some((name, forms)) = headers.first().map(|n| (n.clone(), data)) else {
            continue;
        };
        let forms: Vec<String> = forms.into_iter().map(shorten_pronouns).collect();
        if !name.is_empty() && !forms.is_empty() {
            group.tenses.push(Tense { name, forms });
        }
    }

    groups.retain(|g| !g.tenses.is_empty());
    (!groups.is_empty()).then_some(groups)
}

/// The page has several tables (pronunciation, translations…); the
/// conjugation one is the one that lists the non-personal forms.
fn find_conjugation_table(html: &str) -> Option<&str> {
    let mut i = 0;
    while let Some(at) = find_open(html, i, "table", None) {
        let table = balanced(html, at, "table");
        if table.contains("Formas no personales") || table.contains("Modo indicativo") {
            return Some(table);
        }
        i = at + table.len();
    }
    None
}

/// Rows that only separate blocks or explain the symbols.
fn is_divider(title: &str) -> bool {
    title.is_empty() || title.starts_with("Leyenda") || title == "Formas personales"
}

/// Shortens long pronouns so the forms fit in the terminal.
fn shorten_pronouns(form: String) -> String {
    const PAIRS: &[(&str, &str)] = &[
        ("que él, que ella, que usted", "que él/ella/ud."),
        ("que ustedes, que ellos", "que uds./ellos"),
        ("él, ella, usted", "él/ella/ud."),
        ("ustedes, ellos", "uds./ellos"),
    ];
    let mut out = form;
    for (long, short) in PAIRS {
        if out.starts_with(long) {
            out = format!("{short}{}", &out[long.len()..]);
            break;
        }
    }
    out
}

fn cells(tr: &str, tag: &str) -> Vec<String> {
    let mut out = Vec::new();
    let mut i = 0;
    while let Some(at) = find_open(tr, i, tag, None) {
        let el = balanced(tr, at, tag);
        out.push(strip_tags(el));
        i = at + el.len();
    }
    out
}

#[cfg(test)]
mod tests {
    use super::*;

    const TABLE: &str = r#"
    <table class="flex wikitable">
      <tr><th colspan="8">Formas no personales (verboides)</th></tr>
      <tr><th>Infinitivo</th><td colspan="3">hablar</td><td colspan="4">haber hablado</td></tr>
      <tr><th colspan="8">Formas personales</th></tr>
      <tr><th colspan="8">Modo indicativo</th></tr>
      <tr><th></th><th>yo</th><th>tú</th><th>vos</th></tr>
      <tr><th>Presente</th><td>yo hablo</td><td>tú hablas</td><td>vos hablás</td></tr>
      <tr><th colspan="8">Leyenda: † arcaico</th></tr>
    </table>"#;

    #[test]
    fn builds_groups_and_tenses() {
        let g = parse_table(TABLE).expect("should find the table");
        assert_eq!(
            g.len(),
            2,
            "moods: {:?}",
            g.iter().map(|x| &x.title).collect::<Vec<_>>()
        );
        assert_eq!(g[0].title, "Formas no personales (verboides)");
        assert_eq!(g[0].tenses[0].name, "Infinitivo");
        assert_eq!(g[0].tenses[0].forms, ["hablar", "haber hablado"]);
        assert_eq!(g[1].title, "Modo indicativo");
        assert_eq!(
            g[1].tenses[0].forms,
            ["yo hablo", "tú hablas", "vos hablás"]
        );
    }

    #[test]
    fn skips_dividers_pronoun_row_and_legend() {
        let g = parse_table(TABLE).unwrap();
        let titles: Vec<&str> = g.iter().map(|x| x.title.as_str()).collect();
        assert!(!titles.contains(&"Formas personales"));
        assert!(!titles.iter().any(|t| t.starts_with("Leyenda")));
        // The pronoun row must not become a tense.
        assert!(g[1].tenses.iter().all(|t| !t.name.is_empty()));
        assert_eq!(g[1].tenses.len(), 1);
    }

    #[test]
    fn takes_only_the_spanish_section() {
        // A page with the conjugation in another language, like «casa», a
        // noun in Spanish and a verb in Romanian.
        let page =
            format!("<h2>Español</h2><p>sustantivo</p><h2>Rumano</h2>{TABLE}<h2>Véneto</h2>");
        let es = spanish_section(&page).unwrap();
        assert!(es.contains("sustantivo"));
        assert!(
            !es.contains("Modo indicativo"),
            "another language leaked in"
        );
        assert!(parse_table(es).is_none());
    }

    #[test]
    fn keeps_the_spanish_table_when_it_is_the_one() {
        let page = format!("<h2>Español</h2>{TABLE}<h2>Rumano</h2><p>otra cosa</p>");
        let es = spanish_section(&page).unwrap();
        assert!(parse_table(es).is_some());
        assert!(!es.contains("otra cosa"));
    }

    #[test]
    fn spanish_section_runs_to_the_end_when_it_is_the_only_language() {
        let page = format!("<h2>Español</h2>{TABLE}");
        assert!(parse_table(spanish_section(&page).unwrap()).is_some());
    }

    #[test]
    fn returns_none_when_the_page_has_no_spanish_section() {
        assert!(spanish_section("<h2>Rumano</h2><p>x</p>").is_none());
    }

    #[test]
    fn returns_none_when_there_is_no_conjugation_table() {
        assert!(parse_table("<table><tr><td>pronunciación</td></tr></table>").is_none());
    }

    #[test]
    fn shortens_long_pronouns() {
        assert_eq!(
            shorten_pronouns("él, ella, usted habla".into()),
            "él/ella/ud. habla"
        );
        assert_eq!(