Speed up deck synchronization

Apply changes for what has been inserted, updated, removed.

Also use transactions to speed up multi-writing.

12 files changed, 291 insertions, 135 deletions

diff --git a/src/db/mod.rs b/src/db/mod.rs
index c2749dc..434d74a 100644
--- a/src/db/mod.rs
+++ b/src/db/mod.rs
@@ -1,81 +1,99 @@
-use crate::{
-    model::{card::Card, entry::Entry},
-    space_repetition,
-    util::serialization,
-};
 use anyhow::Result;
 use rusqlite::{params, Connection};
 use rusqlite_migration::{Migrations, M};
 
+use crate::model::DbEntry;
 use crate::util::time;
+use crate::{
+    model::{Card, Question},
+    space_repetition,
+    util::serialization,
+};
 
 pub fn init(database: String) -> Result<Connection> {
     let mut conn = Connection::open(database)?;
     let migrations = Migrations::new(vec![
         M::up(include_str!("sql/1-init.sql")),
         M::up(include_str!("sql/2-primary-key-question-responses.sql")),
+        M::up(include_str!("sql/3-drop-deck-read.sql")),
     ]);
     migrations.to_latest(&mut conn)?;
     Ok(conn)
 }
 
-pub fn last_deck_read(conn: &Connection) -> Option<u64> {
-    let mut stmt = conn
-        .prepare("SELECT deck_read FROM cards ORDER BY deck_read DESC LIMIT 1")
-        .ok()?;
-
-    let mut rows = stmt.query([]).ok()?;
-    let row = rows.next().ok()??;
-    row.get(0).ok()?
+pub fn all(conn: &Connection) -> Result<Vec<DbEntry>> {
+    let mut stmt = conn.prepare("SELECT question, responses, deleted FROM cards")?;
+
+    let res: Result<Vec<DbEntry>, rusqlite::Error> = stmt
+        .query_map([], |row| {
+            let responses: String = row.get(1)?;
+            Ok(DbEntry {
+                question: row.get(0)?,
+                responses: serialization::line_to_words(&responses),
+                deleted: row.get(2)?,
+            })
+        })?
+        .collect();
+
+    Ok(res?)
 }
 
-/// Synchronize the DB with the deck:
-///
-/// - insert new cards,
-/// - keep existing cards,
-/// - hide unused cards (keep state in case the card is added back afterward).
-pub fn synchronize(conn: &Connection, entries: Vec<Entry>) -> Result<()> {
+pub fn insert(conn: &mut Connection, questions: &Vec<Question>) -> Result<()> {
     let now = time::seconds_since_unix_epoch()?;
-
     let state = serde_json::to_string(&space_repetition::init())?;
 
-    for entry in entries {
-        let concat_1 = serialization::words_to_line(&entry.part_1);
-        let concat_2 = serialization::words_to_line(&entry.part_2);
-
-        for w in entry.part_1.iter() {
-            insert(conn, now, w, &concat_2, &state)?;
-        }
-
-        for w in entry.part_2.iter() {
-            insert(conn, now, w, &concat_1, &state)?;
-        }
+    let tx = conn.transaction()?;
+    for Question {
+        question,
+        responses,
+    } in questions
+    {
+        let responses = serialization::words_to_line(responses);
+        tx.execute(
+            "
+            INSERT INTO cards (question, responses, state, created, ready)
+            VALUES (?, ?, ?, ?, ?)
+            ",
+            params![question, responses, state, now, now],
+        )?;
     }
-
-    delete_read_before(conn, now)?;
-
+    tx.commit()?;
     Ok(())
 }
 
-fn insert(conn: &Connection, now: u64, question: &str, responses: &str, state: &str) -> Result<()> {
-    conn.execute(
-        "
-        INSERT INTO cards (question, responses, state, created, deck_read, ready)
-        VALUES (?, ?, ?, ?, ?, ?)
-        ON CONFLICT (question, responses) DO UPDATE SET deck_read = ?, deleted = null
-        ",
-        params![question, responses, state, now, now, now, now],
-    )?;
+pub fn delete(conn: &mut Connection, questions: &Vec<Question>) -> Result<()> {
+    let now = time::seconds_since_unix_epoch()?;
 
+    let tx = conn.transaction()?;
+    for Question {
+        question,
+        responses,
+    } in questions
+    {
+        let responses = serialization::words_to_line(responses);
+        tx.execute(
+            "UPDATE cards SET deleted = ? WHERE question = ? AND responses = ?",
+            params![now, question, responses],
+        )?;
+    }
+    tx.commit()?;
     Ok(())
 }
 
-fn delete_read_before(conn: &Connection, t: u64) -> Result<()> {
-    conn.execute(
-        "UPDATE cards SET deleted = ? WHERE deck_read < ?",
-        params![t, t],
-    )?;
-
+pub fn undelete(conn: &mut Connection, questions: &Vec<Question>) -> Result<()> {
+    let tx = conn.transaction()?;
+    for Question {
+        question,
+        responses,
+    } in questions
+    {
+        let responses = serialization::words_to_line(responses);
+        tx.execute(
+            "UPDATE cards SET deleted = NULL WHERE question = ? AND responses = ?",
+            params![question, responses],
+        )?;
+    }
+    tx.commit()?;
     Ok(())
 }
 
diff --git a/src/db/sql/3-drop-deck-read.sql b/src/db/sql/3-drop-deck-read.sql
new file mode 100644
index 0000000..1ca23d1
--- /dev/null
+++ b/src/db/sql/3-drop-deck-read.sql
@@ -0,0 +1 @@
+ALTER TABLE cards DROP COLUMN deck_read;
diff --git a/src/deck.rs b/src/deck.rs
index d23529f..0c302e1 100644
--- a/src/deck.rs
+++ b/src/deck.rs
@@ -1,4 +1,4 @@
-use crate::{model::entry::Entry, util::serialization};
+use crate::{model::Line, util::serialization};
 use anyhow::{Error, Result};
 use std::fmt;
 use std::fs::File;
@@ -23,10 +23,10 @@ impl std::error::Error for ParseError {
     }
 }
 
-pub fn read(deck: &str) -> Result<Vec<Entry>> {
+pub fn read(deck: &str) -> Result<Vec<Line>> {
     let file = File::open(deck)?;
     let reader = BufReader::new(file);
-    let mut entries: Vec<Entry> = Vec::new();
+    let mut entries: Vec<Line> = Vec::new();
 
     for (index, line) in reader.lines().enumerate() {
         let line = line?;
@@ -57,9 +57,9 @@ pub fn read(deck: &str) -> Result<Vec<Entry>> {
                                 .to_string(),
                         }));
                     } else {
-                        entries.push(Entry {
-                            part_1: serialization::line_to_words(&t1.to_string()),
-                            part_2: serialization::line_to_words(&t2.to_string()),
+                        entries.push(Line {
+                            part_1: serialization::line_to_words(t1),
+                            part_2: serialization::line_to_words(t2),
                         })
                     }
                 }
diff --git a/src/gui/message.rs b/src/gui/message.rs
index 1ae10a5..29b5d8a 100644
--- a/src/gui/message.rs
+++ b/src/gui/message.rs
@@ -27,7 +27,7 @@ pub fn show<B: Backend>(
             let d1 = util::title(title);
             f.render_widget(d1, chunks[0]);
 
-            let message = Paragraph::new(util::center_vertically(chunks[1], &message.to_string()))
+            let message = Paragraph::new(util::center_vertically(chunks[1], message))
                 .alignment(Alignment::Center);
             f.render_widget(message, chunks[1]);
         })?;
diff --git a/src/gui/mod.rs b/src/gui/mod.rs
index 858b30d..358e4b5 100644
--- a/src/gui/mod.rs
+++ b/src/gui/mod.rs
@@ -2,12 +2,10 @@ pub mod message;
 pub mod question;
 pub mod util;
 
-use crate::deck;
-use crate::util::time;
-use crate::{db, space_repetition, util::event::Events};
+use crate::{db, space_repetition, util::event::Events, util::time};
 use anyhow::Result;
 use rusqlite::Connection;
-use std::{fs, io, time::Duration};
+use std::io;
 use termion::{raw::IntoRawMode, raw::RawTerminal, screen::AlternateScreen};
 use tui::{backend::TermionBackend, Terminal};
 
@@ -20,28 +18,6 @@ pub fn terminal() -> Result<Term> {
     Ok(Terminal::new(backend)?)
 }
 
-pub fn synchronize(
-    conn: &Connection,
-    term: &mut Term,
-    events: &Events,
-    deck_path: &str,
-    deck_name: &str,
-) -> Result<()> {
-    let last_modified = time::seconds_since_unix_epoch_of(fs::metadata(deck_path)?.modified()?)?;
-    let last_deck_read = db::last_deck_read(conn);
-    let must_synchronize = last_deck_read.map(|r| r < last_modified).unwrap_or(true);
-
-    if must_synchronize {
-        let _ = message::show(term, events, deck_name, "Synchronization du deck…", false);
-        time::wait_at_least(
-            || db::synchronize(conn, deck::read(deck_path)?),
-            Duration::from_secs(1),
-        )?;
-    }
-
-    Ok(())
-}
-
 pub fn start(conn: &Connection, term: &mut Term, events: &Events, deck_name: &str) -> Result<()> {
     let mut answers = 0;
 
@@ -66,7 +42,7 @@ pub fn start(conn: &Connection, term: &mut Term, events: &Events, deck_name: &st
                     Some(ready) => {
                         let duration = time::pp_duration(ready - now);
                         format!("Prochaine carte disponible dans {duration}.")
-                    },
+                    }
                     None => "Aucune carte n’est disponible. Votre deck est-il vide ?".to_string(),
                 };
                 let _ = message::show(term, events, &title, &message, true);
diff --git a/src/gui/question.rs b/src/gui/question.rs
index 71c3ea2..2aa6e65 100644
--- a/src/gui/question.rs
+++ b/src/gui/question.rs
@@ -1,6 +1,6 @@
 use crate::{
     gui::util,
-    model::{card::Card, difficulty, difficulty::Difficulty},
+    model::{difficulty, difficulty::Difficulty, Card},
     util::event::{Event, Events},
     util::serialization,
 };
@@ -150,7 +150,9 @@ pub fn ask<B: Backend>(
                     Key::Char(c) => {
                         state.input.push(c);
                         if is_correct(&state.input, &card.responses) {
-                            state.answer = Answer::Difficulty { difficulty: Difficulty::Good }
+                            state.answer = Answer::Difficulty {
+                                difficulty: Difficulty::Good,
+                            }
                         }
                     }
                     Key::Backspace => {
@@ -161,12 +163,12 @@ pub fn ask<B: Backend>(
                     }
                     Key::Ctrl('w') => {
                         let mut words = state.input.split_whitespace().collect::<Vec<&str>>();
-                        if words.len() > 0 {
+                        if !words.is_empty() {
                             words.truncate(words.len() - 1);
                             state.input = format!(
                                 "{}{}",
                                 words.join(" "),
-                                if words.len() > 0 { " " } else { "" }
+                                if !words.is_empty() { " " } else { "" }
                             );
                         }
                     }
diff --git a/src/main.rs b/src/main.rs
index 4ca3822..a791f29 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -3,11 +3,11 @@ mod deck;
 mod gui;
 mod model;
 mod space_repetition;
+mod sync;
 mod util;
 
 use crate::util::event::Events;
 use anyhow::Result;
-use rusqlite::Connection;
 use std::path::PathBuf;
 use structopt::StructOpt;
 
@@ -20,11 +20,14 @@ struct Opt {
 
 fn main() -> Result<()> {
     let deck_path = Opt::from_args().deck;
-    let conn = db::init(db_path(&deck_path))?;
+    let mut conn = db::init(db_path(&deck_path))?;
     let deck_name = deck::pp_from_path(&deck_path).unwrap_or_else(|| "Deck".to_string());
+
+    sync::run(&mut conn, &deck_path)?;
+
     let mut term = gui::terminal()?;
     let events = Events::new();
-    match run_tui(conn, &deck_path, &deck_name, &mut term, &events) {
+    match gui::start(&conn, &mut term, &events, &deck_name) {
         Ok(()) => Ok(()),
         Err(msg) => {
             // Show errors in TUI, otherwise they are hidden
@@ -34,17 +37,6 @@ fn main() -> Result<()> {
     }
 }
 
-fn run_tui(
-    conn: Connection,
-    deck_path: &str,
-    deck_name: &str,
-    term: &mut gui::Term,
-    events: &Events,
-) -> Result<()> {
-    gui::synchronize(&conn, term, &events, &deck_path, &deck_name)?;
-    gui::start(&conn, term, &events, &deck_name)
-}
-
 fn db_path(deck_path: &str) -> String {
     let mut path = PathBuf::from(deck_path);
     path.set_extension("db");
diff --git a/src/model/card.rs b/src/model/card.rs
deleted file mode 100644
index 811f877..0000000
--- a/src/model/card.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-use crate::space_repetition;
-
-#[derive(Debug)]
-pub struct Card {
-    pub question: String,
-    pub responses: Vec<String>,
-    pub state: space_repetition::State,
-    pub ready: u64,
-}
diff --git a/src/model/entry.rs b/src/model/entry.rs
deleted file mode 100644
index 769b38c..0000000
--- a/src/model/entry.rs
+++ /dev/null
@@ -1,5 +0,0 @@
-#[derive(Debug, Clone)]
-pub struct Entry {
-    pub part_1: Vec<String>,
-    pub part_2: Vec<String>,
-}
diff --git a/src/model/mod.rs b/src/model/mod.rs
index 185f401..2dc1ab5 100644
--- a/src/model/mod.rs
+++ b/src/model/mod.rs
@@ -1,3 +1,29 @@
-pub mod card;
+use crate::space_repetition;
+
 pub mod difficulty;
-pub mod entry;
+
+#[derive(Debug, Clone)]
+pub struct Line {
+    pub part_1: Vec<String>,
+    pub part_2: Vec<String>,
+}
+
+pub struct DbEntry {
+    pub question: String,
+    pub responses: Vec<String>,
+    pub deleted: Option<u64>,
+}
+
+#[derive(Debug)]
+pub struct Card {
+    pub question: String,
+    pub responses: Vec<String>,
+    pub state: space_repetition::State,
+    pub ready: u64,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct Question {
+    pub question: String,
+    pub responses: Vec<String>,
+}
diff --git a/src/sync.rs b/src/sync.rs
new file mode 100644
index 0000000..3911d55
--- /dev/null
+++ b/src/sync.rs
@@ -0,0 +1,173 @@
+use crate::{
+    db, deck,
+    model::{DbEntry, Line, Question},
+};
+use anyhow::Result;
+use rusqlite::Connection;
+use std::collections::HashSet;
+
+pub fn run(conn: &mut Connection, deck_path: &str) -> Result<()> {
+    let db_entries = db::all(conn)?;
+    let lines = deck::read(deck_path)?;
+    let Diff {
+        new,
+        deleted,
+        undeleted,
+    } = diff(db_entries, lines);
+
+    db::insert(conn, &new)?;
+    db::delete(conn, &deleted)?;
+    db::undelete(conn, &undeleted)?;
+
+    Ok(())
+}
+
+struct Diff {
+    pub new: Vec<Question>,
+    pub deleted: Vec<Question>,
+    pub undeleted: Vec<Question>,
+}
+
+fn diff(db_entries: Vec<DbEntry>, lines: Vec<Line>) -> Diff {
+    let mut file_questions: HashSet<Question> = HashSet::new();
+    let mut db_questions_not_deleted: HashSet<Question> = HashSet::new();
+    let mut db_questions_deleted: HashSet<Question> = HashSet::new();
+
+    for Line { part_1, part_2 } in lines {
+        for question in part_1.clone() {
+            let mut responses = part_2.clone();
+            responses.sort();
+            file_questions.insert(Question {
+                question,
+                responses,
+            });
+        }
+        for question in part_2 {
+            let mut responses = part_1.clone();
+            responses.sort();
+            file_questions.insert(Question {
+                question,
+                responses,
+            });
+        }
+    }
+
+    for DbEntry {
+        question,
+        mut responses,
+        deleted,
+    } in db_entries
+    {
+        responses.sort();
+        if deleted.is_some() {
+            db_questions_deleted.insert(Question {
+                question,
+                responses,
+            });
+        } else {
+            db_questions_not_deleted.insert(Question {
+                question,
+                responses,
+            });
+        }
+    }
+
+    let new = file_questions
+        .difference(&db_questions_not_deleted)
+        .cloned()
+        .collect::<HashSet<Question>>()
+        .difference(&db_questions_deleted)
+        .cloned()
+        .collect();
+
+    let deleted = db_questions_not_deleted
+        .difference(&file_questions)
+        .cloned()
+        .collect();
+
+    let undeleted = file_questions
+        .intersection(&db_questions_deleted)
+        .cloned()
+        .collect();
+
+    Diff {
+        new,
+        deleted,
+        undeleted,
+    }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use super::*;
+
+    #[test]
+    fn sync() {
+        let db_entries = vec![
+            DbEntry {
+                question: "A".to_string(),
+                responses: vec!["A".to_string()],
+                deleted: None,
+            },
+            DbEntry {
+                question: "B".to_string(),
+                responses: vec!["B".to_string()],
+                deleted: None,
+            },
+            DbEntry {
+                question: "C".to_string(),
+                responses: vec!["C".to_string()],
+                deleted: Some(0),
+            },
+            DbEntry {
+                question: "D".to_string(),
+                responses: vec!["D".to_string()],
+                deleted: Some(0),
+            },
+        ];
+
+        let lines = vec![
+            Line {
+                part_1: vec!["A".to_string()],
+                part_2: vec!["A".to_string()],
+            },
+            Line {
+                part_1: vec!["C".to_string()],
+                part_2: vec!["C".to_string()],
+            },
+            Line {
+                part_1: vec!["E".to_string()],
+                part_2: vec!["E".to_string()],
+            },
+        ];
+
+        let Res {
+            new,
+            deleted,
+            undeleted,
+        } = sync(db_entries, lines);
+
+        assert_eq!(
+            new,
+            vec!(Question {
+                question: "E".to_string(),
+                responses: vec!("E".to_string())
+            })
+        );
+        assert_eq!(
+            deleted,
+            vec!(Question {
+                question: "B".to_string(),
+                responses: vec!("B".to_string())
+            })
+        );
+        assert_eq!(
+            undeleted,
+            vec!(Question {
+                question: "C".to_string(),
+                responses: vec!("C".to_string())
+            })
+        );
+    }
+}
diff --git a/src/util/time.rs b/src/util/time.rs
index 679d1b4..e4bf75c 100644
--- a/src/util/time.rs
+++ b/src/util/time.rs
@@ -1,5 +1,4 @@
 use anyhow::Result;
-use std::thread;
 use std::time::SystemTime;
 
 pub fn seconds_since_unix_epoch() -> Result<u64> {
@@ -36,20 +35,3 @@ fn plural(n: u64, str: &str) -> String {
         format!("{n} {str}s")
     }
 }
-
-/// Call the function, then sleep if necessary.
-///
-/// Calling this will at least take the duration asked for in parameters.
-pub fn wait_at_least<F>(f: F, d: std::time::Duration) -> Result<()>
-where
-    F: Fn() -> Result<()>,
-{
-    let t1 = SystemTime::now();
-    f()?;
-    let t2 = SystemTime::now();
-    let elapsed = t2.duration_since(t1)?;
-    if elapsed < d {
-        thread::sleep(d - elapsed);
-    }
-    Ok(())
-}