caching

iota-storage/src/util/chat_files.rs

@@ -1,5 +1,6 @@
 use crate::storage_error::StorageError;
 use crate::util::db;
+use crate::util::sync::{self, EntityType, Operation};
 use iota_logger::log;
 use rusqlite::params;
 
@@ -46,6 +47,7 @@ impl MessageState {
 #[derive(Debug, Clone)]
 pub struct StoredMessage {
     pub id: i64,
+    pub external_user: i64,
     pub message_time: i64,
     pub content: String,
     pub edited: bool,
@@ -150,7 +152,8 @@ fn update_message_content(
             .unwrap()
             .as_millis() as i64;
 
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             r#"
             INSERT INTO message_edits (message_id, content_before, content_after, edited_at, edited_by)
             VALUES (?1, ?2, ?3, ?4, ?5)
@@ -158,7 +161,7 @@ fn update_message_content(
             params![msg_id, old_content, new_content, now, editor_id],
         )?;
 
-        conn.execute(
+        tx.execute(
             r#"
             UPDATE messages
             SET content = ?1, edited_count = edited_count + 1
@@ -166,6 +169,14 @@ fn update_message_content(
             "#,
             params![new_content, msg_id],
         )?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Upsert,
+        )?;
+        tx.commit()?;
 
         Ok(())
     })
@@ -187,15 +198,24 @@ pub fn hard_delete_message(
             |row| row.get(0),
         )?;
 
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             "DELETE FROM message_edits WHERE message_id = ?1",
             params![msg_id],
         )?;
-        conn.execute(
+        tx.execute(
             "DELETE FROM reactions WHERE message_id = ?1",
             params![msg_id],
         )?;
-        conn.execute("DELETE FROM messages WHERE id = ?1", params![msg_id])?;
+        tx.execute("DELETE FROM messages WHERE id = ?1", params![msg_id])?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Delete,
+        )?;
+        tx.commit()?;
         Ok(())
     })
 }
@@ -261,7 +281,8 @@ pub fn flag_deleted_by_external(
     message_time: i64,
 ) -> Result<(), StorageError> {
     db::with_db(|conn| {
-        let affected = conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        let affected = tx.execute(
             r#"
             UPDATE messages
             SET deleted_by_external = 1
@@ -272,6 +293,15 @@ pub fn flag_deleted_by_external(
         if affected == 0 {
             return Err(StorageError::Other("Message not found".into()));
         }
+        let msg_id: i64 = tx.query_row("SELECT id FROM messages WHERE storage_owner = ?1 AND external_user = ?2 AND message_time = ?3 ORDER BY id DESC LIMIT 1", params![storage_owner, external_user, message_time], |r| r.get(0))?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Delete,
+        )?;
+        tx.commit()?;
         Ok(())
     })
 }
@@ -297,10 +327,19 @@ pub fn delete_edit_history(
             |row| row.get(0),
         )?;
 
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             "DELETE FROM message_edits WHERE message_id = ?1",
             params![msg_id],
         )?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Upsert,
+        )?;
+        tx.commit()?;
         Ok(())
     })
 }
@@ -328,13 +367,22 @@ pub fn add_reaction(
             .unwrap()
             .as_millis() as i64;
 
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             r#"
             INSERT OR IGNORE INTO reactions (message_id, user_id, reaction, created_at)
             VALUES (?1, ?2, ?3, ?4)
             "#,
             params![msg_id, user_id, reaction, now],
         )?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Upsert,
+        )?;
+        tx.commit()?;
         Ok(())
     })
 }
@@ -357,10 +405,19 @@ pub fn remove_reaction(
             |row| row.get(0),
         )?;
 
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             "DELETE FROM reactions WHERE message_id = ?1 AND user_id = ?2 AND reaction = ?3",
             params![msg_id, user_id, reaction],
         )?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Upsert,
+        )?;
+        tx.commit()?;
         Ok(())
     })
 }
@@ -383,7 +440,8 @@ pub fn add_message(
     };
 
     if let Err(e) = db::with_db(|conn| {
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             r#"
             INSERT INTO messages (
                 storage_owner, external_user, message_time, content,
@@ -405,6 +463,15 @@ pub fn add_message(
                 reply_to,
             ],
         )?;
+        let msg_id = tx.last_insert_rowid();
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Message,
+            msg_id,
+            Operation::Upsert,
+        )?;
+        tx.commit()?;
         Ok(())
     }) {
         log!("Failed to insert message into sqlite: {}", e);
@@ -447,7 +514,8 @@ pub fn change_message_state(
             .as_str()
             .to_string();
 
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             r#"
             UPDATE messages
             SET message_state = ?1
@@ -459,6 +527,13 @@ pub fn change_message_state(
             "#,
             params![upgraded, storage_owner, external_user, timestamp],
         )?;
+        let msg_id: i64 = tx.query_row(
+            "SELECT id FROM messages WHERE storage_owner = ?1 AND external_user = ?2 AND message_time = ?3 ORDER BY id DESC LIMIT 1",
+            params![storage_owner, external_user, timestamp],
+            |row| row.get(0),
+        )?;
+        sync::record_event(&tx, storage_owner, EntityType::Message, msg_id, Operation::Upsert)?;
+        tx.commit()?;
         Ok(())
     })
     .map_err(|e: StorageError| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))
@@ -532,6 +607,7 @@ pub fn get_messages(
             |row| {
                 Ok(StoredMessage {
                     id: row.get(0)?,
+                    external_user,
                     message_time: row.get(1)?,
                     content: row.get(2)?,
                     sent_by_self: row.get::<_, i64>(3)? != 0,
@@ -568,6 +644,69 @@ pub fn get_messages(
     }
 }
 
+pub fn get_messages_by_ids(storage_owner: i64, ids: &[i64]) -> Vec<StoredMessage> {
+    if ids.is_empty() {
+        return Vec::new();
+    }
+    let wanted: std::collections::HashSet<i64> = ids.iter().copied().collect();
+    // A journal id uniquely identifies a row. Load all messages for this owner and retain only
+    // those ids; this keeps reaction hydration identical to normal message loading.
+    match db::with_db(|conn| {
+        let mut stmt = conn.prepare("SELECT id, message_time, content, sent_by_self, message_state, height, reply_to, edited_count, external_user FROM messages WHERE storage_owner = ?1 AND deleted_by_external = 0")?;
+        let rows = stmt.query_map([storage_owner], |row| {
+            let external_user: i64 = row.get(8)?;
+            Ok(StoredMessage {
+                id: row.get(0)?,
+                external_user,
+                message_time: row.get(1)?,
+                content: row.get(2)?,
+                sent_by_self: row.get::<_, i64>(3)? != 0,
+                message_state: row.get(4)?,
+                height: row.get(5).unwrap_or(0),
+                reply_to: row.get(6).ok().flatten(),
+                edited: row.get::<_, i64>(7).unwrap_or(0) > 0,
+                reactions: Vec::new(),
+            })
+        })?;
+        let mut messages = Vec::new();
+        for row in rows {
+            let message = row?;
+            if wanted.contains(&message.id) {
+                messages.push(message);
+            }
+        }
+        let reaction_map = load_reactions(conn, &messages.iter().map(|m| m.id).collect::<Vec<_>>());
+        for message in &mut messages {
+            message.reactions = reaction_map.get(&message.id).cloned().unwrap_or_default();
+        }
+        Ok(messages)
+    }) {
+        Ok(messages) => messages,
+        Err(e) => {
+            log!("Failed to query messages by id: {}", e);
+            Vec::new()
+        }
+    }
+}
+
+pub fn get_all_messages(storage_owner: i64) -> Vec<StoredMessage> {
+    let ids = match db::with_db(|conn| {
+        let mut stmt = conn.prepare(
+            "SELECT id FROM messages WHERE storage_owner = ?1 AND deleted_by_external = 0",
+        )?;
+        Ok(stmt
+            .query_map([storage_owner], |row| row.get::<_, i64>(0))?
+            .collect::<Result<Vec<_>, _>>()?)
+    }) {
+        Ok(ids) => ids,
+        Err(e) => {
+            log!("Failed to query all messages: {}", e);
+            return Vec::new();
+        }
+    };
+    get_messages_by_ids(storage_owner, &ids)
+}
+
 #[cfg(test)]
 mod tests {
     use super::MessageState;

iota-storage/src/util/chats_util.rs

@@ -1,10 +1,12 @@
 use crate::users::contact::Contact;
 use crate::util::db;
+use crate::util::sync::{self, EntityType, Operation};
 use rusqlite::params;
 
 pub fn mod_user(storage_owner: i64, contact: &Contact) {
     if let Err(e) = db::with_db(|conn| {
-        conn.execute(
+        let tx = conn.unchecked_transaction()?;
+        tx.execute(
             r#"
             INSERT INTO contacts (storage_owner, user_id, user_name, last_message_at)
             VALUES (?1, ?2, ?3, ?4)
@@ -19,12 +21,31 @@ pub fn mod_user(storage_owner: i64, contact: &Contact) {
                 contact.last_message_at,
             ],
         )?;
+        sync::record_event(
+            &tx,
+            storage_owner,
+            EntityType::Contact,
+            contact.user_id,
+            Operation::Upsert,
+        )?;
+        tx.commit()?;
         Ok(())
     }) {
         eprintln!("Failed to mod_user: {}", e);
     }
 }
 
+pub fn get_users_by_ids(storage_owner: i64, ids: &[i64]) -> Vec<Contact> {
+    if ids.is_empty() {
+        return Vec::new();
+    }
+    let wanted: std::collections::HashSet<i64> = ids.iter().copied().collect();
+    get_users(storage_owner)
+        .into_iter()
+        .filter(|contact| wanted.contains(&contact.user_id))
+        .collect()
+}
+
 pub fn get_user(storage_owner: i64, user_id: i64) -> Option<Contact> {
     match db::with_db(|conn| {
         match conn.query_row(

iota-storage/src/util/db.rs

@@ -220,6 +220,37 @@ fn run_migrations_on_connection(conn: &Connection) -> Result<(), StorageError> {
         )?;
     }
 
+    if current_version < 6 {
+        conn.execute_batch(
+            r#"
+            CREATE TABLE IF NOT EXISTS sync_heads (
+                user_id INTEGER PRIMARY KEY,
+                version INTEGER NOT NULL
+            );
+            CREATE TABLE IF NOT EXISTS sync_events (
+                user_id INTEGER NOT NULL,
+                version INTEGER NOT NULL,
+                entity_type TEXT NOT NULL,
+                entity_id INTEGER NOT NULL,
+                operation TEXT NOT NULL,
+                created_at INTEGER NOT NULL,
+                PRIMARY KEY (user_id, version)
+            );
+            CREATE INDEX IF NOT EXISTS idx_sync_events_user_version
+                ON sync_events (user_id, version);
+            CREATE TABLE IF NOT EXISTS client_sync_state (
+                user_id INTEGER NOT NULL,
+                session_id INTEGER NOT NULL,
+                acknowledged_version INTEGER NOT NULL,
+                cache_schema_version INTEGER NOT NULL,
+                updated_at INTEGER NOT NULL,
+                PRIMARY KEY (user_id, session_id)
+            );
+            PRAGMA user_version = 6;
+            "#,
+        )?;
+    }
+
     Ok(())
 }
 
@@ -289,7 +320,7 @@ mod tests {
         run_migrations_on_connection(&conn)?;
 
         let version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?;
-        assert_eq!(version, 5);
+        assert_eq!(version, 6);
         for column in ["height", "reply_to", "edited_count", "deleted_by_external"] {
             let mut statement =
                 conn.prepare("SELECT 1 FROM pragma_table_info('messages') WHERE name = ?1")?;
@@ -298,4 +329,23 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn migrates_version_five_once_and_is_idempotent() -> Result<(), StorageError> {
+        let conn = Connection::open_in_memory()?;
+        conn.execute_batch("PRAGMA user_version = 5;")?;
+        run_migrations_on_connection(&conn)?;
+        run_migrations_on_connection(&conn)?;
+        let version: i64 = conn.pragma_query_value(None, "user_version", |row| row.get(0))?;
+        assert_eq!(version, 6);
+        for table in ["sync_heads", "sync_events", "client_sync_state"] {
+            let exists: i64 = conn.query_row(
+                "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1",
+                [table],
+                |row| row.get(0),
+            )?;
+            assert_eq!(exists, 1);
+        }
+        Ok(())
+    }
 }

iota-storage/src/util/mod.rs

@@ -5,3 +5,4 @@ pub mod config_util;
 pub mod db;
 pub mod e2ee_storage;
 pub mod settings;
+pub mod sync;

iota-storage/src/util/sync.rs

@@ -0,0 +1,160 @@
+//! Durable per-user state journal used by device cache synchronization.
+use crate::storage_error::StorageError;
+use crate::util::db;
+use rusqlite::{Transaction, params};
+use std::collections::BTreeMap;
+
+pub const CACHE_SCHEMA_VERSION: i64 = 1;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EntityType {
+    Message,
+    Contact,
+}
+impl EntityType {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Message => "message",
+            Self::Contact => "contact",
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Operation {
+    Upsert,
+    Delete,
+}
+impl Operation {
+    fn as_str(self) -> &'static str {
+        match self {
+            Self::Upsert => "upsert",
+            Self::Delete => "delete",
+        }
+    }
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct Delta {
+    pub message_upserts: Vec<i64>,
+    pub deleted_message_ids: Vec<i64>,
+    pub contact_upserts: Vec<i64>,
+    pub deleted_contact_ids: Vec<i64>,
+}
+
+pub fn now_millis() -> i64 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .as_millis() as i64
+}
+
+pub fn record_event(
+    tx: &Transaction<'_>,
+    user_id: i64,
+    entity: EntityType,
+    entity_id: i64,
+    operation: Operation,
+) -> Result<i64, StorageError> {
+    tx.execute(
+        "INSERT INTO sync_heads (user_id, version) VALUES (?1, 0) ON CONFLICT(user_id) DO NOTHING",
+        [user_id],
+    )?;
+    let previous: i64 = tx.query_row(
+        "SELECT version FROM sync_heads WHERE user_id = ?1",
+        [user_id],
+        |r| r.get(0),
+    )?;
+    let version = previous
+        .checked_add(1)
+        .ok_or_else(|| StorageError::Other("sync version overflow".into()))?;
+    tx.execute(
+        "UPDATE sync_heads SET version = ?2 WHERE user_id = ?1",
+        params![user_id, version],
+    )?;
+    tx.execute("INSERT INTO sync_events (user_id, version, entity_type, entity_id, operation, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", params![user_id, version, entity.as_str(), entity_id, operation.as_str(), now_millis()])?;
+    Ok(version)
+}
+
+pub fn head(user_id: i64) -> Result<i64, StorageError> {
+    db::with_db(|conn| {
+        Ok(conn
+            .query_row(
+                "SELECT version FROM sync_heads WHERE user_id = ?1",
+                [user_id],
+                |r| r.get(0),
+            )
+            .unwrap_or(0))
+    })
+}
+
+pub fn has_session(user_id: i64, session_id: i64) -> Result<bool, StorageError> {
+    db::with_db(|conn| {
+        Ok(conn
+            .query_row(
+                "SELECT 1 FROM client_sync_state WHERE user_id = ?1 AND session_id = ?2",
+                params![user_id, session_id],
+                |_| Ok(()),
+            )
+            .is_ok())
+    })
+}
+
+pub fn acknowledge(
+    user_id: i64,
+    session_id: i64,
+    version: i64,
+    cache_schema_version: i64,
+) -> Result<(), StorageError> {
+    if user_id <= 0 || session_id <= 0 || version < 0 {
+        return Err(StorageError::Other("invalid sync acknowledgement".into()));
+    }
+    db::with_db(|conn| {
+        let head = conn
+            .query_row(
+                "SELECT version FROM sync_heads WHERE user_id = ?1",
+                [user_id],
+                |r| r.get(0),
+            )
+            .unwrap_or(0);
+        if version > head {
+            return Err(StorageError::Other(
+                "acknowledgement is ahead of head".into(),
+            ));
+        }
+        conn.execute("INSERT INTO client_sync_state (user_id, session_id, acknowledged_version, cache_schema_version, updated_at) VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT(user_id, session_id) DO UPDATE SET acknowledged_version = MAX(acknowledged_version, excluded.acknowledged_version), cache_schema_version = excluded.cache_schema_version, updated_at = excluded.updated_at", params![user_id, session_id, version, cache_schema_version, now_millis()])?;
+        Ok(())
+    })
+}
+
+/// Returns the final operation for each entity after `from_version`.
+pub fn delta(user_id: i64, from_version: i64, captured_head: i64) -> Result<Delta, StorageError> {
+    if from_version < 0 || from_version > captured_head {
+        return Err(StorageError::Other("invalid sync cursor".into()));
+    }
+    db::with_db(|conn| {
+        let mut stmt = conn.prepare("SELECT entity_type, entity_id, operation FROM sync_events WHERE user_id = ?1 AND version > ?2 AND version <= ?3 ORDER BY version ASC")?;
+        let mut final_events = BTreeMap::<(String, i64), String>::new();
+        for row in stmt.query_map(params![user_id, from_version, captured_head], |r| {
+            Ok((
+                r.get::<_, String>(0)?,
+                r.get::<_, i64>(1)?,
+                r.get::<_, String>(2)?,
+            ))
+        })? {
+            let (kind, id, operation) = row?;
+            final_events.insert((kind, id), operation);
+        }
+        let mut out = Delta::default();
+        for ((kind, id), operation) in final_events {
+            match (kind.as_str(), operation.as_str()) {
+                ("message", "delete") => out.deleted_message_ids.push(id),
+                ("message", _) => out.message_upserts.push(id),
+                ("contact", "delete") => out.deleted_contact_ids.push(id),
+                ("contact", _) => out.contact_upserts.push(id),
+                _ => {}
+            }
+        }
+        Ok(out)
+    })
+}