wa-cli: WhatsApp from the terminal, with a neomutt Maildir bridgeHEADmain

1 files changed, 235 insertions, 0 deletions

diff --git a/src/wamail/sync.js b/src/wamail/sync.js
new file mode 100644
index 0000000..ce244e1
--- /dev/null
+++ b/src/wamail/sync.js
@@ -0,0 +1,235 @@
+// wamail sync: WhatsApp (the wa-cli daemon) <-> Maildir, one folder per chat.
+//
+// local -> WhatsApp (diff against the saved state):
+//   read every unread message of a chat -> chats.read    mark one as unread -> chats.unread
+//   F (flag) -> messages.star / unstar                   delete -> local only (not downloaded again)
+// WhatsApp -> local:
+//   new messages (backfill of the last N days the first time), media up to mediaMaxBytes,
+//   read/unread when the remote counter changes, names/archived state for the sidebar.
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { messageToEml } from '../format/mail.js';
+import { writeMailboxes } from './neomutt.js';
+import {
+    chatDirName, expandHome, loadState, saveState, scanDir, setFlags, stanzaFileId, writeMessage,
+} from './store.js';
+
+export const DEFAULTS = {
+    backfillDays: 30,         // first time: messages from the last N days...
+    backfillLimit: 100,       // ...and at most N per chat
+    fetchLimit: 50,
+    mediaMaxBytes: 16 * 1024 * 1024,
+    skipChats: [],            // JIDs to ignore
+};
+
+const SKIP_TYPES = new Set([
+    'e2e_notification', 'notification_template', 'notification', 'gp2', 'protocol', 'ciphertext',
+    'broadcast_notification', 'debug', 'unknown', 'reaction', 'revoked',
+]);
+const SUPPORTED_CHAT = /@(c\.us|g\.us|lid)$/;
+
+export function wamailConfig(account, cfg = {}) {
+    return {
+        maildir: account === 'default' ? '~/Mail/whatsapp' : `~/Mail/whatsapp-${account}`,
+        ...DEFAULTS,
+        ...cfg,
+    };
+}
+
+const flagString = (flags) => [...flags].filter((f) => f === 'S' || f === 'F').sort().join('');
+
+// Message deleted for everyone: remove the local copy and do not download it again.
+export function dropLocal({ account, config, stanzaId }) {
+    const state = loadState(account);
+    const fid = stanzaFileId(stanzaId);
+    const sm = state.msgs[fid];
+    if (!sm) return false;
+    const dir = state.chats[sm.chat]?.dir;
+    const entry = dir && scanDir(path.join(expandHome(config.maildir), dir))[fid];
+    if (entry) fs.rmSync(entry.path, { force: true });
+    state.tombstones[fid] = sm;
+    delete state.msgs[fid];
+    saveState(account, state);
+    return true;
+}
+
+export async function syncAccount({ account, rpc, config, onlyChat, log = () => {}, now = Date.now }) {
+    const root = expandHome(config.maildir);
+    const state = loadState(account);
+    const nowSec = Math.floor(now() / 1000);
+    const skipped = (id) => !SUPPORTED_CHAT.test(id) || config.skipChats.includes(id);
+    const stats = { nuevos: 0, leidos: 0, noLeidos: 0, estrellas: 0, borrados: 0 };
+
+    // ---- chats remotos
+    const remote = (onlyChat ? [await rpc('chats.get', { chatId: onlyChat })] : await rpc('chats.list', {}))
+        .filter((c) => !skipped(c.id));
+    const scope = new Set(onlyChat ? remote.map((c) => c.id) : [...remote.map((c) => c.id), ...Object.keys(state.chats)]);
+    try {
+        const me = (await rpc('auth.status', {})).me;
+        if (me?.id) state.me = me;
+    } catch { /* not critical */ }
+
+    // ---- escaneo local
+    const local = {};
+    const scanChat = (chatId) => {
+        const st = state.chats[chatId];
+        if (!st?.dir) return {};
+        const dir = path.join(root, st.dir);
+        const found = scanDir(dir);
+        for (const [fid, e] of Object.entries(found)) local[fid] = { ...e, chat: chatId, dir };
+        return found;
+    };
+    for (const id of scope) scanChat(id);
+    for (const fid of Object.keys(local)) {
+        if (state.tombstones[fid]) {
+            // deleted by mistake (e.g. a race with neomutt): the file is still there
+            state.msgs[fid] = state.tombstones[fid];
+            delete state.tombstones[fid];
+        }
+    }
+
+    // ---- push: cambios locales -> WhatsApp
+    const failed = new Set();
+    const touched = new Map();   // chatId -> 'read' | 'unread'
+    const perChat = new Map();
+    for (const [fid, sm] of Object.entries(state.msgs)) {
+        if (!scope.has(sm.chat)) continue;
+        let loc = local[fid];
+        if (!loc) {
+            loc = scanChat(sm.chat)[fid] && local[fid];   // second attempt before treating it as deleted
+        }
+        if (!loc || loc.flags.has('T')) {
+            if (loc) fs.rmSync(loc.path, { force: true });
+            state.tombstones[fid] = sm;
+            delete state.msgs[fid];
+            delete local[fid];
+            stats.borrados++;
+            continue;
+        }
+        const agg = perChat.get(sm.chat) ?? { had: 0, now: 0, becameUnread: false };
+        perChat.set(sm.chat, agg);
+        const wasSeen = sm.fl.includes('S');
+        const isSeen = loc.flags.has('S');
+        if (!sm.fromMe) {
+            if (!wasSeen) agg.had++;
+            if (!isSeen) agg.now++;
+            if (wasSeen && !isSeen) agg.becameUnread = true;
+        }
+        if (sm.fl.includes('F') !== loc.flags.has('F')) {
+            try {
+                await rpc(loc.flags.has('F') ? 'messages.star' : 'messages.unstar', { messageId: sm.id });
+                stats.estrellas++;
+            } catch (e) {
+                log(`could not change the star of ${fid}: ${e.message}`);
+                failed.add(fid);
+            }
+        }
+    }
+    for (const [chatId, agg] of perChat) {
+        const name = state.chats[chatId]?.name ?? chatId;
+        try {
+            if (agg.becameUnread) {
+                await rpc('chats.unread', { chatId });
+                touched.set(chatId, 'unread');
+                stats.noLeidos++;
+                log(`${name}: marked as unread`);
+            } else if (agg.had > 0 && agg.now === 0) {
+                await rpc('chats.read', { chatId });
+                touched.set(chatId, 'read');
+                stats.leidos++;
+                log(`${name}: marked as read`);
+            }
+        } catch (e) {
+            log(`${name}: could not sync read state: ${e.message}`);
+            for (const [fid, sm] of Object.entries(state.msgs)) if (sm.chat === chatId) failed.add(fid);
+        }
+    }
+
+    // ---- pull: WhatsApp -> local
+    const takenDirs = () => new Set(Object.values(state.chats).map((c) => c.dir).filter(Boolean));
+    for (const c of remote) {
+        const isNew = !state.chats[c.id];
+        const st = (state.chats[c.id] ||= { dir: null, lastTs: 0, unread: null });
+        Object.assign(st, { name: c.name, isGroup: c.isGroup, archived: c.archived, pinned: c.pinned, timestamp: c.timestamp });
+
+        let msgs = [];
+        try {
+            if (isNew || !st.lastTs) {
+                const cutoff = nowSec - config.backfillDays * 86400;
+                if ((c.timestamp ?? 0) >= cutoff) {
+                    msgs = (await rpc('messages.list', { chatId: c.id, limit: config.backfillLimit }))
+                        .filter((m) => m.timestamp >= cutoff);
+                }
+            } else if ((c.timestamp ?? 0) > st.lastTs || onlyChat) {
+                msgs = await rpc('messages.list', { chatId: c.id, since: st.lastTs, limit: config.fetchLimit });
+            }
+        } catch (e) {
+            log(`${c.name}: could not fetch messages: ${e.message}`);
+            continue;
+        }
+
+        let wrote = 0;
+        let maxTs = st.lastTs;
+        for (const m of msgs.sort((a, b) => a.timestamp - b.timestamp)) {
+            maxTs = Math.max(maxTs, m.timestamp);
+            const fid = stanzaFileId(m.stanzaId);
+            if (local[fid] && !state.msgs[fid]) {
+                // lost or incomplete state: rebuild the entry from the file
+                state.msgs[fid] = { chat: c.id, id: m.id, ts: m.timestamp, fromMe: m.fromMe, fl: flagString(local[fid].flags) };
+            }
+            if (state.msgs[fid] || state.tombstones[fid] || SKIP_TYPES.has(m.type) || m.isStatus) continue;
+
+            st.dir ||= chatDirName(c.id, c.name, takenDirs());
+            const dir = path.join(root, st.dir);
+            let full = m;
+            if (m.hasMedia && (m.media?.size == null || m.media.size <= config.mediaMaxBytes)) {
+                try {
+                    full = await rpc('messages.get', { messageId: m.id, includeMedia: true });
+                } catch (e) {
+                    log(`${c.name}: could not download the media of ${fid}: ${e.message}`);
+                }
+            }
+            const raw = messageToEml({ ...full, chatName: full.chatName || c.name });
+            const entry = writeMessage(dir, { fileId: fid, ts: m.timestamp, raw, seen: m.fromMe });
+            local[fid] = { ...entry, chat: c.id, dir };
+            state.msgs[fid] = { chat: c.id, id: m.id, ts: m.timestamp, fromMe: m.fromMe, fl: m.fromMe ? 'S' : '' };
+            wrote++;
+        }
+        if (wrote) {
+            stats.nuevos += wrote;
+            log(`${c.name}: ${wrote} new message(s)`);
+        }
+        st.lastTs = Math.max(maxTs, c.timestamp ?? 0);
+
+        // Read state from WhatsApp: only if the remote counter changed or messages arrived, so
+        // manual changes are not overwritten (WhatsApp has no "partially read").
+        const unread = c.unreadCount ?? 0;
+        if (touched.has(c.id)) {
+            st.unread = touched.get(c.id) === 'read' ? 0 : -1;
+        } else if (st.dir && (wrote || st.unread !== unread)) {
+            const n = unread < 0 ? 1 : unread;
+            const incoming = Object.entries(local)
+                .filter(([fid, l]) => l.chat === c.id && state.msgs[fid] && !state.msgs[fid].fromMe)
+                .sort((a, b) => a[1].ts - b[1].ts);
+            incoming.forEach(([fid, l], i) => {
+                const seen = i < incoming.length - n;
+                if (seen === l.flags.has('S')) return;
+                const flags = new Set(l.flags);
+                if (seen) flags.add('S');
+                else flags.delete('S');
+                local[fid] = { ...l, ...setFlags(l.dir, fid, l, flags) };
+            });
+            st.unread = unread;
+        }
+    }
+
+    // ---- save: current flags (except those that could not be pushed, to retry)
+    for (const [fid, l] of Object.entries(local)) {
+        const sm = state.msgs[fid];
+        if (sm && !failed.has(fid)) sm.fl = flagString(l.flags);
+    }
+    writeMailboxes(account, root, state);
+    saveState(account, state);
+    return stats;
+}