From fba375075a74eaa40a596fca6f0364845c3299da Mon Sep 17 00:00:00 2001 From: link2xt Date: Mon, 19 Jan 2026 21:26:39 +0000 Subject: [PATCH] feat: truncate long messages when loading instead of saving --- deltachat-jsonrpc/src/api.rs | 14 ++++++++-- src/chat.rs | 11 ++------ src/chat/chat_tests.rs | 3 +++ src/context.rs | 17 ++++++++++++ src/download.rs | 2 +- src/html.rs | 25 ++++++++++++----- src/imap.rs | 3 +-- src/message.rs | 40 +++++++++++++++++++++++++--- src/mimeparser.rs | 8 +----- src/mimeparser/mimeparser_tests.rs | 10 +++---- src/receive_imf/receive_imf_tests.rs | 32 ++++------------------ src/tools.rs | 5 +++- 12 files changed, 106 insertions(+), 64 deletions(-) diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 11c5e59d83..4011300c64 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -23,8 +23,9 @@ use deltachat::ephemeral::Timer; use deltachat::imex; use deltachat::location; use deltachat::message::{ - self, delete_msgs_ex, get_existing_msg_ids, get_msg_read_receipt_count, get_msg_read_receipts, - markseen_msgs, Message, MessageState, MsgId, Viewtype, + self, delete_msgs_ex, dont_truncate_long_messages, get_existing_msg_ids, + get_msg_read_receipt_count, get_msg_read_receipts, markseen_msgs, Message, MessageState, MsgId, + Viewtype, }; use deltachat::peer_channels::{ leave_webxdc_realtime, send_webxdc_realtime_advertisement, send_webxdc_realtime_data, @@ -1490,6 +1491,15 @@ impl CommandApi { MsgId::new(message_id).get_html(&ctx).await } + /// Opt out of truncating long messages when loading. + /// + /// Should be used by the UIs that can handle long text messages. + async fn dont_truncate_long_messages(&self, account_id: u32) -> Result<()> { + let ctx = self.get_context(account_id).await?; + dont_truncate_long_messages(&ctx); + Ok(()) + } + /// get multiple messages in one call, /// if loading one message fails the error is stored in the result object in it's place. /// diff --git a/src/chat.rs b/src/chat.rs index ab4b62cd51..ebc322775e 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -51,7 +51,6 @@ use crate::sync::{self, Sync::*, SyncData}; use crate::tools::{ IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id, create_outgoing_rfc724_mid, get_abs_path, gm2local_offset, normalize_text, time, - truncate_msg_text, }; use crate::webxdc::StatusUpdateSerial; @@ -1895,7 +1894,8 @@ impl Chat { EphemeralTimer::Enabled { duration } => time().saturating_add(duration.into()), }; - let (msg_text, was_truncated) = truncate_msg_text(context, msg.text.clone()).await?; + let msg_text = msg.text.clone(); + let new_mime_headers = if msg.has_html() { msg.param.get(Param::SendHtml).map(|s| s.to_string()) } else { @@ -1908,13 +1908,6 @@ impl Chat { html_part.write_part(cursor).ok(); String::from_utf8_lossy(&buffer).to_string() }); - let new_mime_headers = new_mime_headers.or_else(|| match was_truncated { - // We need to add some headers so that they are stripped before formatting HTML by - // `MsgId::get_html()`, not a part of the actual text. Let's add "Content-Type", it's - // anyway a useful metadata about the stored text. - true => Some("Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text), - false => None, - }); let new_mime_headers = match new_mime_headers { Some(h) => Some(tokio::task::block_in_place(move || { buf_compress(h.as_bytes()) diff --git a/src/chat/chat_tests.rs b/src/chat/chat_tests.rs index a27b706701..b176368d8c 100644 --- a/src/chat/chat_tests.rs +++ b/src/chat/chat_tests.rs @@ -9,6 +9,7 @@ use crate::constants::{ use crate::ephemeral::Timer; use crate::headerdef::HeaderDef; use crate::imex::{ImexMode, has_backup, imex}; +use crate::message; use crate::message::{Message, MessengerMessage, delete_msgs}; use crate::mimeparser::{self, MimeMessage}; use crate::qr::{Qr, check_qr}; @@ -5884,6 +5885,8 @@ async fn test_send_edit_request() -> Result<()> { let forwarded = alice2.get_last_msg().await; assert!(!forwarded.is_edited()); + message::dont_truncate_long_messages(alice); + // If a message is too long after editing, it becomes an HTML message on the receiver side. On // the sender side it's still text so that it can be edited again. static REPEAT_TXT: &str = "this text with 42 chars is just repeated.\n"; diff --git a/src/context.rs b/src/context.rs index 3b94c4e027..e581daf3cc 100644 --- a/src/context.rs +++ b/src/context.rs @@ -225,7 +225,23 @@ impl WeakContext { pub struct InnerContext { /// Blob directory path pub(crate) blobdir: PathBuf, + pub(crate) sql: Sql, + + /// True if long text messages should be truncated + /// and full message HTML added. + /// + /// This should be set for the UIs that cannot handle + /// long messages but can display HTML messages. + /// It is enabled by default. + /// + /// UIs that can handle long messages should call + /// [crate::message::dont_truncate_long_messages] + /// to unset it. + /// + /// Ignored for bots, bots never get truncated messages. + pub(crate) truncate_long_messages: AtomicBool, + /// The global "ongoing" process state. /// /// This is a global mutex-like state for operations which should be modal in the @@ -484,6 +500,7 @@ impl Context { blobdir, running_state: RwLock::new(Default::default()), sql: Sql::new(dbfile), + truncate_long_messages: AtomicBool::new(true), oauth2_mutex: Mutex::new(()), wrong_pw_warning_mutex: Mutex::new(()), housekeeping_mutex: Mutex::new(()), diff --git a/src/download.rs b/src/download.rs index b5f9ec0c1d..1a8b0cf4b9 100644 --- a/src/download.rs +++ b/src/download.rs @@ -209,7 +209,7 @@ impl Session { let (sender, receiver) = async_channel::unbounded(); { let _fetch_msgs_lock_guard = context.fetch_msgs_mutex.lock().await; - self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender) + Box::pin(self.fetch_many_msgs(context, folder, vec![uid], &uid_message_ids, sender)) .await?; } if receiver.recv().await.is_err() { diff --git a/src/html.rs b/src/html.rs index a72c7137ce..12b7e395e4 100644 --- a/src/html.rs +++ b/src/html.rs @@ -32,7 +32,7 @@ impl Message { /// The corresponding ffi-function is `dc_msg_has_html()`. /// To get the HTML-code of the message, use `MsgId.get_html()`. pub fn has_html(&self) -> bool { - self.mime_modified + self.mime_modified || self.full_text.is_some() } /// Set HTML-part part of a message that is about to be sent. @@ -277,7 +277,7 @@ impl MsgId { if let Some(html) = param.get(SendHtml) { return Ok(Some(html.to_string())); } - let from_rawmime = |rawmime: Vec| { + let from_rawmime = async |rawmime: Vec| { if !rawmime.is_empty() { match HtmlMsgParser::from_bytes(context, &rawmime) { Err(err) => { @@ -287,19 +287,30 @@ impl MsgId { Ok((parser, _)) => Ok(Some(parser.html)), } } else { - warn!(context, "get_html: no mime for {}", self); - Ok(None) + let msg = Message::load_from_db(context, self).await?; + if let Some(full_text) = &msg.full_text { + let html = PlainText { + text: full_text.clone(), + flowed: false, + delsp: false, + } + .to_html(); + Ok(Some(html)) + } else { + warn!(context, "get_html: no mime for {}", self); + Ok(None) + } } }; if compressed { - return from_rawmime(buf_decompress(&headers)?); + return from_rawmime(buf_decompress(&headers)?).await; } let headers2 = headers.clone(); let compressed = match tokio::task::block_in_place(move || buf_compress(&headers2)) { Err(e) => { warn!(context, "get_mime_headers: buf_compress() failed: {}", e); - return from_rawmime(headers); + return from_rawmime(headers).await; } Ok(o) => o, }; @@ -324,7 +335,7 @@ WHERE id=? AND mime_headers!='' AND mime_compressed=0", "get_mime_headers: failed to update mime_headers: {}", e ); } - from_rawmime(headers) + from_rawmime(headers).await } } diff --git a/src/imap.rs b/src/imap.rs index 29acf6e8c5..7d1b203708 100644 --- a/src/imap.rs +++ b/src/imap.rs @@ -765,8 +765,7 @@ impl Imap { }; let actually_download_messages_future = async { - session - .fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender) + Box::pin(session.fetch_many_msgs(context, folder, uids_fetch, &uid_message_ids, sender)) .await .context("fetch_many_msgs") }; diff --git a/src/message.rs b/src/message.rs index 4a80b57442..8e2831432f 100644 --- a/src/message.rs +++ b/src/message.rs @@ -3,6 +3,7 @@ use std::collections::BTreeSet; use std::path::{Path, PathBuf}; use std::str; +use std::sync::atomic::Ordering; use anyhow::{Context as _, Result, ensure, format_err}; use deltachat_contact_tools::{VcardContact, parse_vcard}; @@ -32,10 +33,9 @@ use crate::param::{Param, Params}; use crate::reaction::get_msg_reactions; use crate::summary::Summary; use crate::sync::SyncData; -use crate::tools::create_outgoing_rfc724_mid; use crate::tools::{ - get_filebytes, get_filemeta, gm2local_offset, read_file, sanitize_filename, time, - timestamp_to_str, + create_outgoing_rfc724_mid, get_filebytes, get_filemeta, gm2local_offset, read_file, + sanitize_filename, time, timestamp_to_str, truncate_msg_text, }; /// Message ID, including reserved IDs. @@ -441,7 +441,13 @@ pub struct Message { pub(crate) timestamp_rcvd: i64, pub(crate) ephemeral_timer: EphemeralTimer, pub(crate) ephemeral_timestamp: i64, + + /// Message text, possibly truncated if the message is large. pub(crate) text: String, + + /// Full text if the message text is truncated. + pub(crate) full_text: Option, + /// Text that is added to the end of Message.text /// /// Currently used for adding the download information on pre-messages @@ -566,6 +572,7 @@ impl Message { } _ => String::new(), }; + let msg = Message { id: row.get("id")?, rfc724_mid: row.get::<_, String>("rfc724mid")?, @@ -590,6 +597,7 @@ impl Message { original_msg_id: row.get("original_msg_id")?, mime_modified: row.get("mime_modified")?, text, + full_text: None, additional_text: String::new(), subject: row.get("subject")?, param: row.get::<_, String>("param")?.parse().unwrap_or_default(), @@ -607,6 +615,15 @@ impl Message { .with_context(|| format!("failed to load message {id} from the database"))?; if let Some(msg) = &mut msg { + if !msg.mime_modified { + let (truncated_text, was_truncated) = + truncate_msg_text(context, msg.text.clone()).await?; + if was_truncated { + msg.full_text = Some(msg.text.clone()); + msg.text = truncated_text; + } + } + msg.additional_text = Self::get_additional_text(context, msg.download_state, &msg.param)?; } @@ -2342,5 +2359,22 @@ impl Viewtype { } } +/// Opt out of truncating long messages. +/// +/// After calling this function, long messages +/// will not be truncated during loading. +/// +/// UIs should call this function if they +/// can handle long messages by cutting them +/// and displaying "Show full message" option. +/// +/// Has no effect for bots which never +/// truncate messages when loading. +pub fn dont_truncate_long_messages(context: &Context) { + context + .truncate_long_messages + .store(false, Ordering::Relaxed); +} + #[cfg(test)] mod message_tests; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index eda7f8540a..54a6427b2b 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -31,7 +31,7 @@ use crate::message::{self, Message, MsgId, Viewtype, get_vcard_summary, set_msg_ use crate::param::{Param, Params}; use crate::simplify::{SimplifiedText, simplify}; use crate::sync::SyncItems; -use crate::tools::{get_filemeta, parse_receive_headers, time, truncate_msg_text, validate_id}; +use crate::tools::{get_filemeta, parse_receive_headers, time, validate_id}; use crate::{chatlist_events, location, tools}; /// Public key extracted from `Autocrypt-Gossip` @@ -1438,12 +1438,6 @@ impl MimeMessage { (simplified_txt, top_quote) }; - let (simplified_txt, was_truncated) = - truncate_msg_text(context, simplified_txt).await?; - if was_truncated { - self.is_mime_modified = was_truncated; - } - if !simplified_txt.is_empty() || simplified_quote.is_some() { let mut part = Part { dehtml_failed, diff --git a/src/mimeparser/mimeparser_tests.rs b/src/mimeparser/mimeparser_tests.rs index 562ea644a4..7b2f9945f7 100644 --- a/src/mimeparser/mimeparser_tests.rs +++ b/src/mimeparser/mimeparser_tests.rs @@ -1287,12 +1287,12 @@ async fn test_mime_modified_large_plain() -> Result<()> { { let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; - assert!(mimemsg.is_mime_modified); - assert!( - mimemsg.parts[0].msg.matches("just repeated").count() - <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len() + assert!(!mimemsg.is_mime_modified); + assert!(mimemsg.parts[0].msg.matches("just repeated").count() == REPEAT_CNT); + assert_eq!( + mimemsg.parts[0].msg.len() + 1, + REPEAT_TXT.len() * REPEAT_CNT ); - assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len()); } for draft in [false, true] { diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs index cc6c8af5b5..b17293665b 100644 --- a/src/receive_imf/receive_imf_tests.rs +++ b/src/receive_imf/receive_imf_tests.rs @@ -3610,39 +3610,17 @@ async fn test_big_forwarded_with_big_attachment() -> Result<()> { .starts_with("this text with 42 chars is just repeated.") ); assert!(msg.get_text().ends_with("[...]")); - assert!(!msg.has_html()); - - let msg = Message::load_from_db(bob, rcvd.msg_ids[2]).await?; - assert_eq!(msg.get_viewtype(), Viewtype::File); assert!(msg.has_html()); let html = msg.id.get_html(bob).await?.unwrap(); - let tail = html - .split_once("Hello!") - .unwrap() - .1 - .split_once("From: AAA") - .unwrap() - .1 - .split_once("aaa@example.org") - .unwrap() - .1 - .split_once("To: Alice") - .unwrap() - .1 - .split_once("alice@example.org") - .unwrap() - .1 - .split_once("Subject: Some subject") - .unwrap() - .1 - .split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000") - .unwrap() - .1; assert_eq!( - tail.matches("this text with 42 chars is just repeated.") + html.matches("this text with 42 chars is just repeated.") .count(), 128 ); + + let msg = Message::load_from_db(bob, rcvd.msg_ids[2]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::File); + assert!(!msg.has_html()); Ok(()) } diff --git a/src/tools.rs b/src/tools.rs index cfa31781aa..ba9126d19c 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -9,6 +9,7 @@ use std::mem; use std::ops::{AddAssign, Deref}; use std::path::{Path, PathBuf}; use std::str::from_utf8; +use std::sync::atomic::Ordering; // If a time value doesn't need to be sent to another host, saved to the db or otherwise used across // program restarts, a monotonically nondecreasing clock (`Instant`) should be used. But as // `Instant` may use `libc::clock_gettime(CLOCK_MONOTONIC)`, e.g. on Android, and does not advance @@ -139,7 +140,9 @@ pub(crate) fn truncate_by_lines( /// /// Returns the resulting text and a bool telling whether a truncation was done. pub(crate) async fn truncate_msg_text(context: &Context, text: String) -> Result<(String, bool)> { - if context.get_config_bool(Config::Bot).await? { + if !context.truncate_long_messages.load(Ordering::Relaxed) + || context.get_config_bool(Config::Bot).await? + { return Ok((text, false)); } // Truncate text if it has too many lines